ruby-kafka 0.3.8 → 0.3.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ca28feeee3d3fd46dab6145eb79a67b842d7f6ec
-  data.tar.gz: 720984362d2a5c4492a35022749d458618010736
+  metadata.gz: 0681733f79e04de3118e211640474ed752c8b340
+  data.tar.gz: d35d33238ec84ad331c7af167ac3464c29bc65ff
 SHA512:
-  metadata.gz: c95bc7e7befbf91326d87a3a21af6bed1ba138b2243849f071084b1ec351eae728e0d03da20581f20a47a1b11a1b8842495aaf5ac55b8d6530a8c17c6e219a45
-  data.tar.gz: 56b6062281cec53d293a3e4e2e295b423dc8fbc9e2affef51f67c5c5f6697daf064f07bf6a6b34d3cd7b94161dffcfe050cdfc3e967cbe58a317ba041cfc84ef
+  metadata.gz: 7b040e4e49d312961a23f4b42238877e0c014734c4f99de3df41f0c2db011afe12f3dd91d7edaaa016131c28925233c27482fd1d55a7207eeb2e767f2996151c
+  data.tar.gz: cb3b411ff48f4ae6868adbec7f122e6793d50ea93cb8acfd7c5228ab20ef023e43a50b136249939154649b75dc4b09b6ea0c611e6011ef9c79b2d01c17d63edb

data/.gitignore

@@ -1,6 +1,5 @@
 /.bundle/
 /.yardoc
-/Gemfile.lock
 /_yardoc/
 /coverage/
 /doc/

data/CHANGELOG.md

@@ -4,6 +4,14 @@ Changes and additions to the library will be listed here.
 
 ## Unreleased
 
+## v0.3.9
+
+- Improve the default durability setting. The producer setting `required_acks` now defaults to `:all` (#210).
+- Handle rebalances in the producer (#196). *Mpampis Kostas*
+- Add simplified producer and consumer APIs for simple use cases.
+- Add out-of-the-box Datadog reporting.
+- Improve producer performance.
+
 ## v0.3.8
 
 - Keep separate connection pools for consumers and producers initialized from

data/Gemfile.lock

@@ -0,0 +1,81 @@
+PATH
+  remote: .
+  specs:
+    ruby-kafka (0.3.8)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (4.2.5)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    benchmark-perf (0.1.0)
+    builder (3.2.2)
+    coderay (1.1.0)
+    colored (1.2)
+    diff-lcs (1.2.5)
+    docker-api (1.25.0)
+      excon (>= 0.38.0)
+      json
+    dogstatsd-ruby (1.6.0)
+    dotenv (2.1.0)
+    excon (0.45.4)
+    i18n (0.7.0)
+    json (1.8.3)
+    method_source (0.8.2)
+    minitest (5.8.3)
+    pry (0.9.12.6)
+      coderay (~> 1.0)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    rake (10.5.0)
+    rspec (3.4.0)
+      rspec-core (~> 3.4.0)
+      rspec-expectations (~> 3.4.0)
+      rspec-mocks (~> 3.4.0)
+    rspec-benchmark (0.1.0)
+      benchmark-perf (~> 0.1.0)
+      rspec (>= 3.0.0, < 4.0.0)
+    rspec-core (3.4.1)
+      rspec-support (~> 3.4.0)
+    rspec-expectations (3.4.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.4.0)
+    rspec-mocks (3.4.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.4.0)
+    rspec-support (3.4.1)
+    rspec_junit_formatter (0.2.2)
+      builder (< 4)
+      rspec-core (>= 2, < 4, != 2.12.0)
+    ruby-prof (0.15.9)
+    slop (3.6.0)
+    snappy (0.0.12)
+    thread_safe (0.3.5)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activesupport
+  bundler (>= 1.9.5)
+  colored
+  docker-api
+  dogstatsd-ruby
+  dotenv
+  pry
+  rake (~> 10.0)
+  rspec
+  rspec-benchmark
+  rspec_junit_formatter (= 0.2.2)
+  ruby-kafka!
+  ruby-prof
+  snappy
+
+BUNDLED WITH
+   1.10.6

data/Procfile

@@ -0,0 +1,2 @@
+producer: bundle exec ruby ci/producer.rb
+consumer: bundle exec ruby ci/consumer.rb

data/README.md

@@ -9,25 +9,31 @@ Although parts of this library work with Kafka 0.8 – specifically, the Produce
 ## Table of Contents
 
 1. [Installation](#installation)
-2. [Usage](#usage)
-  1. [Producing Messages to Kafka](#producing-messages-to-kafka)
+2. [Compatibility](#compatibility)
+3. [Usage](#usage)
+  1. [Setting up the Kafka Client](#setting-up-the-kafka-client)
+  2. [Producing Messages to Kafka](#producing-messages-to-kafka)
+    1. [Efficiently Producing Messages](#efficiently-producing-messages)
     1. [Asynchronously Producing Messages](#asynchronously-producing-messages)
     2. [Serialization](#serialization)
     3. [Partitioning](#partitioning)
     4. [Buffering and Error Handling](#buffering-and-error-handling)
-    5. [Message Delivery Guarantees](#message-delivery-guarantees)
-    6. [Compression](#compression)
-    7. [Producing Messages from a Rails Application](#producing-messages-from-a-rails-application)
-  2. [Consuming Messages from Kafka](#consuming-messages-from-kafka)
+    5. [Message Durability](#message-durability)
+    6. [Message Delivery Guarantees](#message-delivery-guarantees)
+    7. [Compression](#compression)
+    8. [Producing Messages from a Rails Application](#producing-messages-from-a-rails-application)
+  3. [Consuming Messages from Kafka](#consuming-messages-from-kafka)
     1. [Consumer Checkpointing](#consumer-checkpointing)
-    2. [Consuming Messages in Batches](#consuming-messages-in-batches)
-  3. [Thread Safety](#thread-safety)
-  4. [Logging](#logging)
-  5. [Instrumentation](#instrumentation)
-  6. [Understanding Timeouts](#understanding-timeouts)
-  7. [Encryption and Authentication using SSL](#encryption-and-authentication-using-ssl)
-3. [Development](#development)
-4. [Roadmap](#roadmap)
+    2. [Topic Subscriptions](#topic-subscriptions)
+    3. [Consuming Messages in Batches](#consuming-messages-in-batches)
+    4. [Balancing Throughput and Latency](#balancing-throughput-and-latency)
+  4. [Thread Safety](#thread-safety)
+  5. [Logging](#logging)
+  6. [Instrumentation](#instrumentation)
+  7. [Understanding Timeouts](#understanding-timeouts)
+  8. [Encryption and Authentication using SSL](#encryption-and-authentication-using-ssl)
+4. [Development](#development)
+5. [Roadmap](#roadmap)
 
 ## Installation
 
@@ -45,21 +51,76 @@ Or install it yourself as:
 
     $ gem install ruby-kafka
 
+## Compatibility
+
+<table>
+  <tr>
+    <th></th>
+    <th>Kafka 0.8</th>
+    <th>Kafka 0.9</th>
+    <th>Kafka 0.10</th>
+  </tr>
+  <tr>
+    <th>Producer API</th>
+    <td>Full support</td>
+    <td>Full support</td>
+    <td>Limited support</td>
+  </tr>
+  <tr>
+    <th>Consumer API</th>
+    <td>Unsupported</td>
+    <td>Full support</td>
+    <td>Limited support</td>
+  </tr>
+</table>
+
+This library is targeting Kafka 0.9, although there is limited support for versions 0.8 and 0.10:
+
+- **Kafka 0.8:** Full support for the Producer API, but no support for consumer groups. Simple message fetching works.
+- **Kafka 0.10:** Full support for both the Producer and Consumer APIs, but the addition of message timestamps is not supported. However, ruby-kafka should be completely compatible with Kafka 0.10 brokers.
+
+This library requires Ruby 2.1 or higher.
+
 ## Usage
 
 Please see the [documentation site](http://www.rubydoc.info/gems/ruby-kafka) for detailed documentation on the latest release. Note that the documentation on GitHub may not match the version of the library you're using – there are still being made many changes to the API.
 
-### Producing Messages to Kafka
+### Setting up the Kafka Client
 
-A client must be initialized with at least one Kafka broker. Each client keeps a separate pool of broker connections. Don't use the same client from more than one thread.
+A client must be initialized with at least one Kafka broker, from which the entire Kafka cluster will be discovered. Each client keeps a separate pool of broker connections. Don't use the same client from more than one thread.
 
 ```ruby
 require "kafka"
 
-kafka = Kafka.new(seed_brokers: ["kafka1:9092", "kafka2:9092"])
+kafka = Kafka.new(
+  # At least one of these nodes must be available:
+  seed_brokers: ["kafka1:9092", "kafka2:9092"],
+  
+  # Set an optional client id in order to identify the client to Kafka:
+  client_id: "my-application",
+)
 ```
 
-A producer buffers messages and sends them to the broker that is the leader of the partition a given message is assigned to.
+### Producing Messages to Kafka
+
+The simplest way to write a message to a Kafka topic is to call `#deliver_message`:
+
+```ruby
+kafka = Kafka.new(...)
+kafka.deliver_message("Hello, World!", topic: "greetings")
+```
+
+This will write the message to a random partition in the `greetings` topic.
+
+#### Efficiently Producing Messages
+
+While `#deliver_message` works fine for infrequent writes, there are a number of downside:
+
+* Kafka is optimized for transmitting _batches_ of messages rather than individual messages, so there's a significant overhead and performance penalty in using the single-message API.
+* The message delivery can fail in a number of different ways, but this simplistic API does not provide automatic retries.
+* The message is not buffered, so if there is an error, it is lost.
+
+The Producer API solves all these problems and more:
 
 ```ruby
 producer = kafka.producer
@@ -113,8 +174,9 @@ producer.produce("hello", topic: "greetings")
 # `#deliver_messages` will return immediately.
 producer.deliver_messages
 
-# Make sure to call `#shutdown` on the producer in order to
-# avoid leaking resources.
+# Make sure to call `#shutdown` on the producer in order to avoid leaking
+# resources. `#shutdown` will wait for any pending messages to be delivered
+# before returning.
 producer.shutdown
 ```
 
@@ -140,6 +202,8 @@ producer.produce("hello", topic: "greetings")
 # ...
 ```
 
+When calling `#shutdown`, the producer will attempt to deliver the messages and the method call will block until that has happened. Note that there's no _guarantee_ that the messages will be delivered.
+
 **Note:** if the calling thread produces messages faster than the producer can write them to Kafka, you'll eventually run into problems. The internal queue used for sending messages from the calling thread to the background worker has a size limit; once this limit is reached, a call to `#produce` will raise `Kafka::BufferOverflow`.
 
 #### Serialization
@@ -231,6 +295,28 @@ Note that there's a maximum buffer size; pass in a different value for `max_buff
 
 A final note on buffers: local buffers give resilience against broker and network failures, and allow higher throughput due to message batching, but they also trade off consistency guarantees for higher availibility and resilience. If your local process dies while messages are buffered, those messages will be lost. If you require high levels of consistency, you should call `#deliver_messages` immediately after `#produce`.
 
+#### Message Durability
+
+Once the client has delivered a set of messages to a Kafka broker the broker will forward them to its replicas, thus ensuring that a single broker failure will not result in message loss. However, the client can choose _when the leader acknowledges the write_. At one extreme, the client can choose fire-and-forget delivery, not even bothering to check whether the messages have been acknowledged. At the other end, the client can ask the broker to wait until _all_ its replicas have acknowledged the write before returning. This is the safest option, and the default. It's also possible to have the broker return as soon as it has written the messages to its own log but before the replicas have done so. This leaves a window of time where a failure of the leader will result in the messages being lost, although this should not be a common occurence.
+
+Write latency and throughput are negativaly impacted by having more replicas acknowledge a write, so if you require low-latency, high throughput writes you may want to accept lower durability.
+
+This behavior is controlled by the `required_acks` option to `#producer` and `#async_producer`:
+
+```ruby
+# This is the default: all replicas must acknowledge.
+producer = kafka.producer(required_acks: :all)
+
+# This is fire-and-forget: messages can easily be lost.
+producer = kafka.producer(required_acks: 0)
+
+# This only waits for the leader to acknowledge.
+producer = kafka.producer(required_acks: 1)
+```
+
+Unless you absolutely need lower latency it's highly recommended to use the default setting (`:all`).
+
+
 #### Message Delivery Guarantees
 
 There are basically two different and incompatible guarantees that can be made in a message delivery system such as Kafka:
@@ -369,7 +455,10 @@ While this is great for extremely simple use cases, there are a number of downsi
 - If you want to have multiple processes consume from the same topic, there's no way of coordinating which processes should fetch from which partitions.
 - If a process dies, there's no way to have another process resume fetching from the point in the partition that the original process had reached.
 
-The Consumer API solves all of these issues, and more. It uses the Consumer Groups feature released in Kafka 0.9 to allow multiple consumer processes to coordinate access to a topic, assigning each partition to a single consumer. When a consumer fails, the partitions that were assigned to it are re-assigned to other members of the group.
+
+#### Consumer Groups
+
+The Consumer API solves all of the above issues, and more. It uses the Consumer Groups feature released in Kafka 0.9 to allow multiple consumer processes to coordinate access to a topic, assigning each partition to a single consumer. When a consumer fails, the partitions that were assigned to it are re-assigned to other members of the group.
 
 Using the API is simple:
 
@@ -415,6 +504,21 @@ consumer = kafka.consumer(
 ```
 
 
+#### Topic Subscriptions
+
+For each topic subscription it's possible to decide whether to consume messages starting at the beginning of the topic or to just consume new messages that are produced to the topic. This policy is configured by setting the `start_from_beginning` argument when calling `#subscribe`:
+
+```ruby
+# Consume messages from the very beginning of the topic. This is the default.
+consumer.subscribe("users", start_from_beginning: true)
+
+# Only consume new messages.
+consumer.subscribe("notifications", start_from_beginning: false)
+```
+
+Once the consumer group has checkpointed its progress in the topic's partitions, the consumers will always start from the checkpointed offsets, regardless of `start_from_beginning`. As such, this setting only applies when the consumer initially starts consuming from a topic.
+
+
 #### Consuming Messages in Batches
 
 Sometimes it is easier to deal with messages in batches rather than individually. A _batch_ is a sequence of one or more Kafka messages that all belong to the same topic and partition. One common reason to want to use batches is when some external system has a batch or transactional API.
@@ -444,6 +548,32 @@ end
 
 One important thing to note is that the client commits the offset of the batch's messages only after the _entire_ batch has been processed.
 
+
+#### Balancing Throughput and Latency
+
+There are two performance properties that can at times be at odds: _throughput_ and _latency_. Throughput is the number of messages that can be processed in a given timespan; latency is the time it takes from a message is written to a topic until it has been processed.
+
+In order to optimize for throughput, you want to make sure to fetch as many messages as possible every time you do a round trip to the Kafka cluster. This minimizes network overhead and allows processing data in big chunks.
+
+In order to optimize for low latency, you want to process a message as soon as possible, even if that means fetching a smaller batch of messages.
+
+There are two values that can be tuned in order to balance these two concerns: `min_bytes` and `max_wait_time`.
+
+* `min_bytes` is the minimum number of bytes to return from a single message fetch. By setting this to a high value you can increase the processing throughput. The default value is one byte.
+* `max_wait_time` is the maximum number of seconds to wait before returning data from a single message fetch. By setting this high you also increase the processing throughput – and by setting it low you set a bound on latency. This configuration overrides `min_bytes`, so you'll _always_ get data back within the time specified. The default value is five seconds.
+
+Both settings can be passed to either `#each_message` or `#each_batch`, e.g.
+
+```ruby
+# Waits for data for up to 30 seconds, preferring to fetch at least 5KB at a time.
+consumer.each_message(min_bytes: 1024 * 5, max_wait_time: 30) do |message|
+  # ...
+end
+```
+
+If you want to have at most one second of latency, set `max_wait_time: 1`.
+
+
 ### Thread Safety
 
 You typically don't want to share a Kafka client between threads, since the network communication is not synchronized. Furthermore, you should avoid using threads in a consumer unless you're very careful about waiting for all work to complete before returning from the `#each_message` or `#each_batch` block. This is because _checkpointing_ assumes that returning from the block means that the messages that have been yielded have been successfully processed.

data/ci/consumer.rb

@@ -0,0 +1,18 @@
+# Consumes messages from a Kafka topic.
+
+require_relative "init"
+
+consumer = $kafka.consumer(group_id: "greetings-group")
+consumer.subscribe("greetings")
+
+num_messages = 0
+
+trap("TERM") { consumer.stop }
+
+consumer.each_message do |message|
+  num_messages += 1
+
+  if num_messages % 1000 == 0
+    puts "Processed #{num_messages} messages"
+  end
+end

data/ci/init.rb

@@ -0,0 +1,17 @@
+$LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
+
+require "kafka"
+
+logger = Logger.new(STDOUT)
+logger.level = Logger::INFO
+logger.formatter = -> (_, _, _, msg) { msg }
+
+STDOUT.sync = true
+
+$kafka = Kafka.new(
+  logger: logger,
+  seed_brokers: ENV.fetch("HEROKU_KAFKA_URL"),
+  ssl_ca_cert: ENV.fetch("HEROKU_KAFKA_TRUSTED_CERT"),
+  ssl_client_cert: ENV.fetch("HEROKU_KAFKA_CLIENT_CERT"),
+  ssl_client_cert_key: ENV.fetch("HEROKU_KAFKA_CLIENT_CERT_KEY"),
+)

data/ci/producer.rb

@@ -0,0 +1,25 @@
+# Continuously produces messages to a Kafka topic.
+
+require_relative "init"
+
+producer = $kafka.async_producer(
+  delivery_interval: 1,
+  max_queue_size: 5_000,
+  max_buffer_size: 10_000,
+)
+
+num_messages = 0
+shutdown = false
+
+trap("TERM") { shutdown = true }
+
+until shutdown
+  begin
+    producer.produce("hello", key: "world", topic: "greetings")
+  rescue Kafka::BufferOverflow
+    puts "Buffer overflow, backing off..."
+    sleep 10
+  end
+end
+
+producer.shutdown

data/lib/kafka/client.rb

@@ -59,13 +59,64 @@ module Kafka
       @cluster = initialize_cluster
     end
 
+    def deliver_message(value, key: nil, topic:, partition: nil, partition_key: nil)
+      create_time = Time.now
+
+      message = PendingMessage.new(
+        value,
+        key,
+        topic,
+        partition,
+        partition_key,
+        create_time,
+        key.to_s.bytesize + value.to_s.bytesize
+      )
+
+      if partition.nil?
+        partition_count = @cluster.partitions_for(topic).count
+        partition = Partitioner.partition_for_key(partition_count, message)
+      end
+
+      buffer = MessageBuffer.new
+
+      buffer.write(
+        value: message.value,
+        key: message.key,
+        topic: message.topic,
+        partition: partition,
+        create_time: message.create_time,
+      )
+
+      @cluster.add_target_topics([topic])
+
+      compressor = Compressor.new(
+        instrumenter: @instrumenter,
+      )
+
+      operation = ProduceOperation.new(
+        cluster: @cluster,
+        buffer: buffer,
+        required_acks: 1,
+        ack_timeout: 10,
+        compressor: compressor,
+        logger: @logger,
+        instrumenter: @instrumenter,
+      )
+
+      operation.execute
+
+      unless buffer.empty?
+        raise DeliveryFailed
+      end
+    end
+
     # Initializes a new Kafka producer.
     #
     # @param ack_timeout [Integer] The number of seconds a broker can wait for
     #   replicas to acknowledge a write before responding with a timeout.
     #
-    # @param required_acks [Integer] The number of replicas that must acknowledge
-    #   a write.
+    # @param required_acks [Integer, Symbol] The number of replicas that must acknowledge
+    #   a write, or `:all` if all in-sync replicas must acknowledge.
     #
     # @param max_retries [Integer] the number of retries that should be attempted
     #   before giving up sending messages to the cluster. Does not include the
@@ -89,7 +140,7 @@ module Kafka
     #   are per-partition rather than per-topic or per-producer.
     #
     # @return [Kafka::Producer] the Kafka producer.
-    def producer(compression_codec: nil, compression_threshold: 1, ack_timeout: 5, required_acks: 1, max_retries: 2, retry_backoff: 1, max_buffer_size: 1000, max_buffer_bytesize: 10_000_000)
+    def producer(compression_codec: nil, compression_threshold: 1, ack_timeout: 5, required_acks: :all, max_retries: 2, retry_backoff: 1, max_buffer_size: 1000, max_buffer_bytesize: 10_000_000)
       compressor = Compressor.new(
         codec_name: compression_codec,
         threshold: compression_threshold,
@@ -152,6 +203,10 @@ module Kafka
     def consumer(group_id:, session_timeout: 30, offset_commit_interval: 10, offset_commit_threshold: 0, heartbeat_interval: 10)
       cluster = initialize_cluster
 
+      instrumenter = DecoratingInstrumenter.new(@instrumenter, {
+        group_id: group_id,
+      })
+
       group = ConsumerGroup.new(
         cluster: cluster,
         logger: @logger,
@@ -174,7 +229,7 @@ module Kafka
       Consumer.new(
         cluster: cluster,
         logger: @logger,
-        instrumenter: @instrumenter,
+        instrumenter: instrumenter,
         group: group,
         offset_manager: offset_manager,
         session_timeout: session_timeout,
@@ -256,6 +311,32 @@ module Kafka
       operation.execute.flat_map {|batch| batch.messages }
     end
 
+    # EXPERIMENTAL: Enumerates all messages in a topic.
+    def each_message(topic:, offset: :earliest, max_wait_time: 5, min_bytes: 1, max_bytes: 1048576, &block)
+      offsets = Hash.new { offset }
+
+      loop do
+        operation = FetchOperation.new(
+          cluster: @cluster,
+          logger: @logger,
+          min_bytes: min_bytes,
+          max_wait_time: max_wait_time,
+        )
+
+        @cluster.partitions_for(topic).map(&:partition_id).each do |partition|
+          partition_offset = offsets[partition]
+          operation.fetch_from_partition(topic, partition, offset: partition_offset, max_bytes: max_bytes)
+        end
+
+        batches = operation.execute
+
+        batches.each do |batch|
+          batch.messages.each(&block)
+          offsets[batch.partition] = batch.last_offset
+        end
+      end
+    end
+
     # Lists all topics in the cluster.
     #
     # @return [Array<String>] the list of topic names.
@@ -324,12 +405,10 @@ module Kafka
       end
       brokers = []
       seed_brokers.each do |connection|
-        if connection =~ /:\/\//
-          u = URI.parse(connection)
-          brokers << "#{u.host}:#{u.port}"
-        else
-          brokers << connection
-        end
+        connection.prepend("kafka://") unless connection =~ /:\/\//
+        uri = URI.parse(connection)
+        uri.port ||= 9092 # Default Kafka port.
+        brokers << uri
       end
       brokers
     end

data/lib/kafka/cluster.rb

@@ -13,7 +13,7 @@ module Kafka
     #
     # The cluster will try to fetch cluster metadata from one of the brokers.
     #
-    # @param seed_brokers [Array<String>]
+    # @param seed_brokers [Array<URI>]
     # @param broker_pool [Kafka::BrokerPool]
     # @param logger [Logger]
     def initialize(seed_brokers:, broker_pool:, logger:)
@@ -136,10 +136,7 @@ module Kafka
         @logger.info "Fetching cluster metadata from #{node}"
 
         begin
-          host, port = node.split(":", 2)
-          port ||= 9092 # Default Kafka port.
-
-          broker = @broker_pool.connect(host, port.to_i)
+          broker = @broker_pool.connect(node.hostname, node.port)
           cluster_info = broker.fetch_metadata(topics: @target_topics)
 
           @stale = false

data/lib/kafka/compressor.rb

@@ -20,7 +20,7 @@ module Kafka
     # @param codec_name [Symbol, nil]
     # @param threshold [Integer] the minimum number of messages in a message set
     #   that will trigger compression.
-    def initialize(codec_name:, threshold:, instrumenter:)
+    def initialize(codec_name: nil, threshold: 1, instrumenter:)
       @codec = Compression.find_codec(codec_name)
       @threshold = threshold
       @instrumenter = instrumenter

data/lib/kafka/consumer.rb

@@ -65,9 +65,16 @@ module Kafka
     #
     # @param topic [String] the name of the topic to subscribe to.
     # @param default_offset [Symbol] whether to start from the beginning or the
-    #   end of the topic's partitions.
+    #   end of the topic's partitions. Deprecated.
+    # @param start_from_beginning [Boolean] whether to start from the beginning
+    #   of the topic or just subscribe to new messages being produced. This
+    #   only applies when first consuming a topic partition – once the consumer
+    #   has checkpointed its progress, it will always resume from the last
+    #   checkpoint.
     # @return [nil]
-    def subscribe(topic, default_offset: :earliest)
+    def subscribe(topic, default_offset: nil, start_from_beginning: true)
+      default_offset ||= start_from_beginning ? :earliest : :latest
+
       @group.subscribe(topic)
       @offset_manager.set_default_offset(topic, default_offset)
 
@@ -154,6 +161,7 @@ module Kafka
               notification.update(
                 topic: batch.topic,
                 partition: batch.partition,
+                offset_lag: batch.offset_lag,
                 highwater_mark_offset: batch.highwater_mark_offset,
                 message_count: batch.messages.count,
               )

data/lib/kafka/datadog.rb

@@ -0,0 +1,208 @@
+begin
+  require "statsd"
+rescue LoadError
+  $stderr.puts "In order to report Kafka client metrics to Datadog you need to install the `dogstatsd-ruby` gem."
+  raise
+end
+
+require "active_support/subscriber"
+
+module Kafka
+
+  # Reports operational metrics to a Datadog agent using the modified Statsd protocol.
+  #
+  #     require "kafka/datadog"
+  #
+  #     # Default is "ruby_kafka".
+  #     Kafka::Datadog.namespace = "custom-namespace"
+  #
+  #     # Default is "127.0.0.1".
+  #     Kafka::Datadog.host = "statsd.something.com"
+  #
+  #     # Default is 8125.
+  #     Kafka::Datadog.port = 1234
+  #
+  # Once the file has been required, no further configuration is needed – all operational
+  # metrics are automatically emitted.
+  module Datadog
+    STATSD_NAMESPACE = "ruby_kafka"
+
+    def self.statsd
+      @statsd ||= Statsd.new(Statsd::DEFAULT_HOST, Statsd::DEFAULT_HOST, namespace: STATSD_NAMESPACE)
+    end
+
+    def self.host=(host)
+      statsd.host = host
+    end
+
+    def self.port=(port)
+      statsd.port = port
+    end
+
+    def self.namespace=(namespace)
+      statsd.namespace = namespace
+    end
+
+    class StatsdSubscriber < ActiveSupport::Subscriber
+      private
+
+      %w[increment histogram count timing gauge].each do |type|
+        define_method(type) do |*args|
+          emit(type, *args)
+        end
+      end
+
+      def emit(type, *args, tags: {})
+        tags = tags.map {|k, v| "#{k}:#{v}" }.to_a
+
+        Kafka::Datadog.statsd.send(type, *args, tags: tags)
+      end
+    end
+
+    class ConnectionSubscriber < StatsdSubscriber
+      def request(event)
+        client = event.payload.fetch(:client_id)
+        api = event.payload.fetch(:api, "unknown")
+        request_size = event.payload.fetch(:request_size, 0)
+        response_size = event.payload.fetch(:response_size, 0)
+        broker = event.payload.fetch(:broker_host)
+
+        tags = {
+          client: client,
+          api: api,
+          broker: broker
+        }
+
+        timing("api.latency", event.duration, tags: tags)
+        increment("api.calls", tags: tags)
+
+        histogram("api.request_size", request_size, tags: tags)
+        histogram("api.response_size", response_size, tags: tags)
+
+        if event.payload.key?(:exception)
+          increment("api.errors", tags: tags)
+        end
+      end
+
+      attach_to "connection.kafka"
+    end
+
+    class ConsumerSubscriber < StatsdSubscriber
+      def process_message(event)
+        lag = event.payload.fetch(:offset_lag)
+
+        tags = {
+          client: event.payload.fetch(:client_id),
+          topic: event.payload.fetch(:topic),
+          partition: event.payload.fetch(:partition),
+        }
+
+        if event.payload.key?(:exception)
+          increment("consumer.process_message.errors", tags: tags)
+        else
+          timing("consumer.process_message.latency", event.duration, tags: tags)
+          increment("consumer.messages", tags: tags)
+        end
+
+        gauge("consumer.lag", lag, tags: tags)
+      end
+
+      def process_batch(event)
+        messages = event.payload.fetch(:message_count)
+
+        tags = {
+          client: event.payload.fetch(:client_id),
+          topic: event.payload.fetch(:topic),
+          partition: event.payload.fetch(:partition),
+        }
+
+        if event.payload.key?(:exception)
+          increment("consumer.process_batch.errors", tags: tags)
+        else
+          timing("consumer.process_batch.latency", event.duration, tags: tags)
+          count("consumer.messages", messages, tags: tags)
+        end
+      end
+
+      attach_to "consumer.kafka"
+    end
+
+    class ProducerSubscriber < StatsdSubscriber
+      def produce_message(event)
+        client = event.payload.fetch(:client_id)
+        topic = event.payload.fetch(:topic)
+        buffer_size = event.payload.fetch(:buffer_size)
+        max_buffer_size = event.payload.fetch(:max_buffer_size)
+        buffer_fill_ratio = buffer_size.to_f / max_buffer_size.to_f
+
+        tags = {
+          client: client,
+        }
+
+        # This gets us the write rate.
+        increment("producer.produce.messages", tags: tags.merge(topic: topic))
+
+        # This gets us the avg/max buffer size per producer.
+        histogram("producer.buffer.size", buffer_size, tags: tags)
+
+        # This gets us the avg/max buffer fill ratio per producer.
+        histogram("producer.buffer.fill_ratio", buffer_fill_ratio, tags: tags)
+      end
+
+      def buffer_overflow(event)
+        tags = {
+          client: event.payload.fetch(:client_id),
+          topic: event.payload.fetch(:topic),
+        }
+
+        increment("producer.produce.errors", tags: tags)
+      end
+
+      def deliver_messages(event)
+        client = event.payload.fetch(:client_id)
+        message_count = event.payload.fetch(:delivered_message_count)
+        attempts = event.payload.fetch(:attempts)
+
+        tags = {
+          client: client,
+        }
+
+        if event.payload.key?(:exception)
+          increment("producer.deliver.errors", tags: tags)
+        end
+
+        timing("producer.deliver.latency", event.duration, tags: tags)
+
+        # Messages delivered to Kafka:
+        count("producer.deliver.messages", message_count, tags: tags)
+
+        # Number of attempts to deliver messages:
+        histogram("producer.deliver.attempts", attempts, tags: tags)
+      end
+
+      def ack_message(event)
+        tags = {
+          client: event.payload.fetch(:client_id),
+          topic: event.payload.fetch(:topic),
+        }
+
+        # Number of messages ACK'd for the topic.
+        increment("producer.ack.messages", tags: tags)
+
+        # Histogram of delay between a message being produced and it being ACK'd.
+        histogram("producer.ack.delay", event.payload.fetch(:delay), tags: tags)
+      end
+
+      def topic_error(event)
+        tags = {
+          client: event.payload.fetch(:client_id),
+          topic: event.payload.fetch(:topic)
+        }
+
+        increment("producer.ack.errors", tags: tags)
+      end
+
+      attach_to "producer.kafka"
+    end
+  end
+end

data/lib/kafka/fetch_operation.rb

@@ -90,7 +90,7 @@ module Kafka
           }
         }
       }
-    rescue Kafka::LeaderNotAvailable, Kafka::NotLeaderForPartition
+    rescue Kafka::ConnectionError, Kafka::LeaderNotAvailable, Kafka::NotLeaderForPartition
       @cluster.mark_as_stale!
 
       raise

data/lib/kafka/fetched_batch.rb

@@ -24,5 +24,17 @@ module Kafka
     def empty?
       @messages.empty?
     end
+
+    def last_offset
+      messages.last.offset
+    end
+
+    def offset_lag
+      if empty?
+        0
+      else
+        highwater_mark_offset - last_offset
+      end
+    end
   end
 end

data/lib/kafka/instrumenter.rb

@@ -22,4 +22,15 @@ module Kafka
       end
     end
   end
+
+  class DecoratingInstrumenter
+    def initialize(backend, extra_payload = {})
+      @backend = backend
+      @extra_payload = extra_payload
+    end
+
+    def instrument(event_name, payload = {}, &block)
+      @backend.instrument(event_name, @extra_payload.merge(payload), &block)
+    end
+  end
 end

data/lib/kafka/pending_message.rb

@@ -1,18 +1,11 @@
 module Kafka
-  class PendingMessage
-    attr_reader :value, :key, :topic, :partition, :partition_key
-
-    attr_reader :bytesize, :create_time
-
-    def initialize(value:, key:, topic:, partition:, partition_key:, create_time:)
-      @key = key
-      @value = value
-      @topic = topic
-      @partition = partition
-      @partition_key = partition_key
-      @create_time = create_time
-
-      @bytesize = key.to_s.bytesize + value.to_s.bytesize
-    end
-  end
+  PendingMessage = Struct.new(
+    "PendingMessage",
+    :value,
+    :key,
+    :topic,
+    :partition,
+    :partition_key,
+    :create_time,
+    :bytesize)
 end

data/lib/kafka/produce_operation.rb

@@ -138,6 +138,7 @@ module Kafka
           @logger.error "Corrupt message when writing to #{topic}/#{partition}"
         rescue Kafka::UnknownTopicOrPartition
           @logger.error "Unknown topic or partition #{topic}/#{partition}"
+          @cluster.mark_as_stale!
         rescue Kafka::LeaderNotAvailable
           @logger.error "Leader currently not available for #{topic}/#{partition}"
           @cluster.mark_as_stale!

data/lib/kafka/producer.rb

@@ -134,7 +134,7 @@ module Kafka
       @cluster = cluster
       @logger = logger
       @instrumenter = instrumenter
-      @required_acks = required_acks
+      @required_acks = required_acks == :all ? -1 : required_acks
       @ack_timeout = ack_timeout
       @max_retries = max_retries
       @retry_backoff = retry_backoff
@@ -185,12 +185,13 @@ module Kafka
       create_time = Time.now
 
       message = PendingMessage.new(
-        value: value,
-        key: key,
-        topic: topic,
-        partition: partition,
-        partition_key: partition_key,
-        create_time: create_time,
+        value,
+        key,
+        topic,
+        partition,
+        partition_key,
+        create_time,
+        key.to_s.bytesize + value.to_s.bytesize
       )
 
       if buffer_size >= @max_buffer_size

data/lib/kafka/socket_with_timeout.rb

@@ -62,6 +62,8 @@ module Kafka
       end
 
       @socket.read(num_bytes)
+    rescue IO::EAGAINWaitReadable
+      retry
     end
 
     # Writes bytes to the socket, possible with a timeout.

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.3.8"
+  VERSION = "0.3.9"
 end

data/performance/profile.rb

@@ -0,0 +1,39 @@
+$LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
+$LOAD_PATH.unshift(File.expand_path("../../spec", __FILE__))
+
+require "kafka"
+require "ruby-prof"
+require "dotenv"
+require "test_cluster"
+
+Dotenv.load
+
+# Number of times do iterate.
+N = 10_000
+
+KAFKA_CLUSTER = TestCluster.new
+KAFKA_CLUSTER.start
+
+logger = Logger.new(nil)
+
+kafka = Kafka.new(
+  seed_brokers: KAFKA_CLUSTER.kafka_hosts,
+  client_id: "test",
+  logger: logger,
+)
+
+producer = kafka.producer(
+  max_buffer_size: 100_000,
+)
+
+RubyProf.start
+
+N.times do
+  producer.produce("hello", topic: "greetings")
+end
+
+result = RubyProf.stop
+printer = RubyProf::FlatPrinter.new(result)
+printer.print(STDOUT)
+
+KAFKA_CLUSTER.stop

data/ruby-kafka.gemspec

@@ -34,8 +34,10 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "dotenv"
   spec.add_development_dependency "docker-api"
   spec.add_development_dependency "rspec-benchmark"
-  spec.add_development_dependency "activesupport", ">= 4.2.0", "< 5.1"
+  spec.add_development_dependency "activesupport"
   spec.add_development_dependency "snappy"
   spec.add_development_dependency "colored"
   spec.add_development_dependency "rspec_junit_formatter", "0.2.2"
+  spec.add_development_dependency "dogstatsd-ruby"
+  spec.add_development_dependency "ruby-prof"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.3.8
+  version: 0.3.9
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-05-23 00:00:00.000000000 Z
+date: 2016-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -114,20 +114,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.2.0
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '5.1'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 4.2.0
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '5.1'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: snappy
   requirement: !ruby/object:Gem::Requirement
@@ -170,6 +164,34 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.2.2
+- !ruby/object:Gem::Dependency
+  name: dogstatsd-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-prof
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: |-
   A client library for the Kafka distributed commit log.
 
@@ -185,11 +207,16 @@ files:
 - ".yardopts"
 - CHANGELOG.md
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
+- Procfile
 - README.md
 - Rakefile
 - bin/console
 - bin/setup
+- ci/consumer.rb
+- ci/init.rb
+- ci/producer.rb
 - circle.yml
 - examples/firehose-consumer.rb
 - examples/firehose-producer.rb
@@ -208,6 +235,7 @@ files:
 - lib/kafka/connection_builder.rb
 - lib/kafka/consumer.rb
 - lib/kafka/consumer_group.rb
+- lib/kafka/datadog.rb
 - lib/kafka/fetch_operation.rb
 - lib/kafka/fetched_batch.rb
 - lib/kafka/fetched_message.rb
@@ -257,6 +285,7 @@ files:
 - lib/kafka/ssl_socket_with_timeout.rb
 - lib/kafka/version.rb
 - lib/ruby-kafka.rb
+- performance/profile.rb
 - ruby-kafka.gemspec
 homepage: https://github.com/zendesk/ruby-kafka
 licenses: