ruby-kafka 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2842846dbe233b635e3dae0c3e14d9d660d19089
-  data.tar.gz: f7a9b7a9aea3df0e3175b0b239fe9d72f743fed5
+  metadata.gz: c742496ae66067502319aecdfd06536baf455636
+  data.tar.gz: c5b8d1c26990d1d00460907fb4d5838d75d64635
 SHA512:
-  metadata.gz: a4c354e1677e76d0d0eb5483fd56b9d0fba7b8d16baf0f1fe450960977e8c18edba2c00d0a650168d280d03784170e078ae77e7914b6451bd20fc1784aea4867
-  data.tar.gz: 25e871eb218789b094a167371560db89edb1be38387969c2c74fa892701c527abb9d1cd9be0b14b41b65e4805cbbab19dd729c56c963b7555f6d7e15136ea20d
+  metadata.gz: 70392fc8aed22ceabdfaeef0f4d41bc076537edc4a188d927999ab4ecdca8b1c9169d8441e6a2d0b7f2cbea136e8e87fa13fb05ef5d1cb68699770c24ba28acf
+  data.tar.gz: cac553e7b994d4e4bce613d1fa08f6757d82f088e6add7c8f6cce51233901882d4f6297e54890fb7852afd5fd9a64a356faa8bfe17baa1ebc70132a74eca18c8

data/CHANGELOG.md

@@ -4,6 +4,10 @@ Changes and additions to the library will be listed here.
 
 ## Unreleased
 
+## v0.3.2
+
+- Experimental batch consumer API.
+
 ## v0.3.1
 
 - Simplify the heartbeat algorithm.

data/README.md

@@ -17,10 +17,12 @@ Although parts of this library work with Kafka 0.8 – specifically, the Produce
     4. [Buffering and Error Handling](#buffering-and-error-handling)
     5. [Message Delivery Guarantees](#message-delivery-guarantees)
     6. [Compression](#compression)
+    7. [Producing Messages from a Rails Application](#producing-messages-from-a-rails-application)
   2. [Consuming Messages from Kafka](#consuming-messages-from-kafka)
   3. [Logging](#logging)
-  4. [Understanding Timeouts](#understanding-timeouts)
-  5. [Encryption and Authentication using SSL](#encryption-and-authentication-using-ssl)
+  4. [Instrumentation](#instrumentation)
+  5. [Understanding Timeouts](#understanding-timeouts)
+  6. [Encryption and Authentication using SSL](#encryption-and-authentication-using-ssl)
 3. [Development](#development)
 4. [Roadmap](#roadmap)
 
@@ -275,6 +277,52 @@ producer = kafka.producer(
 )
 ```
 
+#### Producing Messages from a Rails Application
+
+A typical use case for Kafka is tracking events that occur in web applications. Oftentimes it's advisable to avoid having a hard dependency on Kafka being available, allowing your application to survive a Kafka outage. By using an asynchronous producer, you can avoid doing IO within the individual request/response cycles, instead pushing that to the producer's internal background thread.
+
+In this example, a producer is configured in a Rails initializer:
+
+```ruby
+# config/initializers/kafka_producer.rb
+require "kafka"
+
+# Configure the Kafka client with the broker hosts and the Rails
+# logger.
+$kafka = Kafka.new(
+  seed_brokers: ["kafka1:9092", "kafka2:9092"],
+  logger: Rails.logger,
+)
+
+# Set up an asynchronous producer that delivers its buffered messages
+# every ten seconds:
+$kafka_producer = $kafka.async_producer(
+  delivery_interval: 10,
+)
+
+# Make sure to shut down the producer when exiting.
+at_exit { $kafka_producer.shutdown }
+```
+
+In your controllers, simply call the producer directly:
+
+```ruby
+# app/controllers/orders_controller.rb
+class OrdersController
+  def create
+    @order = Order.create!(params[:order])
+    
+    event = {
+      order_id: @order.id,
+      amount: @order.amount,
+      timestamp: Time.now,
+    }
+    
+    $kafka_producer.produce(event.to_json, topic: "order_events")
+  end
+end
+```
+
 ### Consuming Messages from Kafka
 
 **Warning:** The Consumer API is still alpha level and will likely change. The consumer code should not be considered stable, as it hasn't been exhaustively tested in production environments yet.
@@ -311,17 +359,14 @@ kafka = Kafka.new(seed_brokers: ["kafka1:9092", "kafka2:9092"])
 # Consumers with the same group id will form a Consumer Group together.
 consumer = kafka.consumer(group_id: "my-consumer")
 
+# It's possible to subscribe to multiple topics by calling `subscribe`
+# repeatedly.
 consumer.subscribe("greetings")
 
-begin
-  # This will loop indefinitely, yielding each message in turn.
-  consumer.each_message do |message|
-    puts message.topic, message.partition
-    puts message.offset, message.key, message.value
-  end
-ensure
-  # Always make sure to shut down the consumer properly.
-  consumer.shutdown
+# This will loop indefinitely, yielding each message in turn.
+consumer.each_message do |message|
+  puts message.topic, message.partition
+  puts message.offset, message.key, message.value
 end
 ```
 
@@ -339,6 +384,47 @@ kafka = Kafka.new(logger: logger, ...)
 
 By default, nothing is logged.
 
+### Instrumentation
+
+Most operations are instrumented using [Active Support Notifications](http://api.rubyonrails.org/classes/ActiveSupport/Notifications.html). In order to subscribe to notifications, make sure to require the notifications library _before_ you require ruby-kafka, e.g.
+
+```ruby
+require "active_support/notifications"
+require "kafka"
+```
+
+The notifications are namespaced based on their origin, with separate namespaces for the producer and the consumer.
+
+In order to receive notifications you can either subscribe to individual notification names or use regular expressions to subscribe to entire namespaces. This example will subscribe to _all_ notifications sent by ruby-kafka:
+
+```ruby
+ActiveSupport::Notifications.subscribe(/.*\.kafka$/) do |*args|
+  event = ActiveSupport::Notifications::Event.new(*args)
+  puts "Received notification `#{event.name}` with payload: #{event.payload.inspect}"
+end
+```
+
+#### Producer Notifications
+
+* `produce_message.producer.kafka` is sent whenever a message is produced to a buffer. It includes the following payload:
+  * `value` is the message value.
+  * `key` is the message key.
+  * `topic` is the topic that the message was produced to.
+  * `buffer_size` is the size of the producer buffer after adding the message.
+  * `max_buffer_size` is the maximum size of the producer buffer.
+
+* `deliver_messages.producer.kafka` is sent whenever a producer attempts to deliver its buffered messages to the Kafka brokers. It includes the following payload:
+  * `attempts` is the number of times delivery was attempted.
+  * `message_count` is the number of messages for which delivery was attempted.
+  * `delivered_message_count` is the number of messages that were acknowledged by the brokers - if this number is smaller than `message_count` not all messages were successfully delivered.
+
+#### Connection Notifications
+
+* `request.connection.kafka` is sent whenever a network request is sent to a Kafka broker. It includes the following payload:
+  * `api` is the name of the API that was called, e.g. `produce` or `fetch`.
+  * `request_size` is the number of bytes in the request.
+  * `response_size` is the number of bytes in the response.
+
 ### Understanding Timeouts
 
 It's important to understand how timeouts work if you have a latency sensitive application. This library allows configuring timeouts on different levels:
@@ -405,7 +491,7 @@ After checking out the repo, run `bin/setup` to install dependencies. Then, run
 
 The current stable release is v0.2. This release is running in production at Zendesk, but it's still not recommended that you use it when data loss is unacceptable. It will take a little while until all edge cases have been uncovered and handled.
 
-### v0.3
+### v0.4
 
 Beta release of the Consumer API, allowing balanced Consumer Groups coordinating access to partitions. Kafka 0.9 only.
 

data/examples/firehose-consumer.rb

@@ -35,20 +35,16 @@ threads = NUM_THREADS.times.map do |worker_id|
     consumer = kafka.consumer(group_id: "firehose")
     consumer.subscribe(KAFKA_TOPIC)
 
-    begin
-      i = 0
-      consumer.each_message do |message|
-        i += 1
+    i = 0
+    consumer.each_message do |message|
+      i += 1
 
-        if i % 1000 == 0
-          queue << i
-          i = 0
-        end
-
-        sleep 0.01
+      if i % 1000 == 0
+        queue << i
+        i = 0
       end
-    ensure
-      consumer.shutdown
+
+      sleep 0.01
     end
   end
 end

data/lib/kafka/client.rb

@@ -235,7 +235,7 @@ module Kafka
 
       operation.fetch_from_partition(topic, partition, offset: offset, max_bytes: max_bytes)
 
-      operation.execute
+      operation.execute.flat_map {|batch| batch.messages }
     end
 
     # Lists all topics in the cluster.

data/lib/kafka/compressor.rb

@@ -34,7 +34,7 @@ module Kafka
 
       wrapper_message = Protocol::Message.new(
         value: compressed_data,
-        attributes: @codec.codec_id,
+        codec_id: @codec.codec_id,
       )
 
       Protocol::MessageSet.new(messages: [wrapper_message])

data/lib/kafka/consumer.rb

@@ -31,22 +31,14 @@ module Kafka
   #     # Subscribe to a Kafka topic:
   #     consumer.subscribe("messages")
   #
-  #     begin
-  #       # Loop forever, reading in messages from all topics that have been
-  #       # subscribed to.
-  #       consumer.each_message do |message|
-  #         puts message.topic
-  #         puts message.partition
-  #         puts message.key
-  #         puts message.value
-  #         puts message.offset
-  #       end
-  #     ensure
-  #       # Make sure to shut down the consumer after use. This lets
-  #       # the consumer notify the Kafka cluster that it's leaving
-  #       # the group, causing a synchronization and re-balancing of
-  #       # the group.
-  #       consumer.shutdown
+  #     # Loop forever, reading in messages from all topics that have been
+  #     # subscribed to.
+  #     consumer.each_message do |message|
+  #       puts message.topic
+  #       puts message.partition
+  #       puts message.key
+  #       puts message.value
+  #       puts message.offset
   #     end
   #
   class Consumer
@@ -60,6 +52,9 @@ module Kafka
 
       # Send two heartbeats in each session window, just to be sure.
       @heartbeat_interval = @session_timeout / 2
+
+      # Whether or not the consumer is currently consuming messages.
+      @running = false
     end
 
     # Subscribes the consumer to a topic.
@@ -94,27 +89,81 @@ module Kafka
     # @yieldparam message [Kafka::FetchedMessage] a message fetched from Kafka.
     # @return [nil]
     def each_message
+      @running = true
+
+      while @running
+        begin
+          fetch_batches.each do |batch|
+            batch.messages.each do |message|
+              Instrumentation.instrument("process_message.consumer.kafka") do |notification|
+                notification.update(
+                  topic: message.topic,
+                  partition: message.partition,
+                  offset: message.offset,
+                  offset_lag: batch.highwater_mark_offset - message.offset,
+                  key: message.key,
+                  value: message.value,
+                )
+
+                yield message
+              end
+
+              @offset_manager.commit_offsets_if_necessary
+
+              send_heartbeat_if_necessary
+              mark_message_as_processed(message)
+
+              break if !@running
+            end
+          end
+        rescue ConnectionError => e
+          @logger.error "Connection error while sending heartbeat; rejoining"
+          join_group
+        rescue UnknownMemberId
+          @logger.error "Kicked out of group; rejoining"
+          join_group
+        rescue RebalanceInProgress
+          @logger.error "Group is rebalancing; rejoining"
+          join_group
+        rescue IllegalGeneration
+          @logger.error "Group has transitioned to a new generation; rejoining"
+          join_group
+        end
+      end
+    ensure
+      # In order to quickly have the consumer group re-balance itself, it's
+      # important that members explicitly tell Kafka when they're leaving.
+      @offset_manager.commit_offsets
+      @group.leave
+      @running = false
+    end
+
+    def stop
+      @running = false
+    end
+
+    def each_batch
       loop do
         begin
-          batch = fetch_batch
-
-          batch.each do |message|
-            Instrumentation.instrument("process_message.consumer.kafka") do |notification|
-              notification.update(
-                topic: message.topic,
-                partition: message.partition,
-                offset: message.offset,
-                key: message.key,
-                value: message.value,
-              )
-
-              yield message
+          fetch_batches.each do |batch|
+            unless batch.empty?
+              Instrumentation.instrument("process_batch.consumer.kafka") do |notification|
+                notification.update(
+                  topic: batch.topic,
+                  partition: batch.partition,
+                  highwater_mark_offset: batch.highwater_mark_offset,
+                  message_count: batch.messages.count,
+                )
+
+                yield batch
+              end
+
+              mark_message_as_processed(batch.messages.last)
             end
 
             @offset_manager.commit_offsets_if_necessary
 
             send_heartbeat_if_necessary
-            mark_message_as_processed(message)
           end
         rescue ConnectionError => e
           @logger.error "Connection error while sending heartbeat; rejoining"
@@ -130,24 +179,11 @@ module Kafka
           join_group
         end
       end
-    end
-
-    # Shuts down the consumer.
-    #
-    # In order to quickly have the consumer group re-balance itself, it's
-    # important that members explicitly tell Kafka when they're leaving.
-    # Therefore it's a good idea to call this method whenever your consumer
-    # is about to quit. If this method is not called, it may take up to
-    # the amount of time defined by the `session_timeout` parameter for
-    # Kafka to realize that this consumer is no longer present and trigger
-    # a group re-balance. In that period of time, the partitions that used
-    # to be assigned to this consumer won't be processed.
-    #
-    # @return [nil]
-    def shutdown
+    ensure
+      # In order to quickly have the consumer group re-balance itself, it's
+      # important that members explicitly tell Kafka when they're leaving.
       @offset_manager.commit_offsets
       @group.leave
-    rescue ConnectionError
     end
 
     private
@@ -157,11 +193,9 @@ module Kafka
       @group.join
     end
 
-    def fetch_batch
+    def fetch_batches
       join_group unless @group.member?
 
-      @logger.debug "Fetching a batch of messages"
-
       assigned_partitions = @group.assigned_partitions
 
       send_heartbeat_if_necessary
@@ -179,17 +213,13 @@ module Kafka
         partitions.each do |partition|
           offset = @offset_manager.next_offset_for(topic, partition)
 
-          @logger.debug "Fetching from #{topic}/#{partition} starting at offset #{offset}"
+          @logger.debug "Fetching batch from #{topic}/#{partition} starting at offset #{offset}"
 
           operation.fetch_from_partition(topic, partition, offset: offset)
         end
       end
 
-      messages = operation.execute
-
-      @logger.info "Fetched #{messages.count} messages"
-
-      messages
+      operation.execute
     rescue ConnectionError => e
       @logger.error "Connection error while fetching messages: #{e}"
 

data/lib/kafka/fetch_operation.rb

@@ -1,3 +1,5 @@
+require "kafka/fetched_batch"
+
 module Kafka
 
   # Fetches messages from one or more partitions.
@@ -66,10 +68,10 @@ module Kafka
         response = broker.fetch_messages(**options)
 
         response.topics.flat_map {|fetched_topic|
-          fetched_topic.partitions.flat_map {|fetched_partition|
+          fetched_topic.partitions.map {|fetched_partition|
             Protocol.handle_error(fetched_partition.error_code)
 
-            fetched_partition.messages.map {|message|
+            messages = fetched_partition.messages.map {|message|
               FetchedMessage.new(
                 value: message.value,
                 key: message.key,
@@ -78,6 +80,13 @@ module Kafka
                 offset: message.offset,
               )
             }
+
+            FetchedBatch.new(
+              topic: fetched_topic.name,
+              partition: fetched_partition.partition,
+              highwater_mark_offset: fetched_partition.highwater_mark_offset,
+              messages: messages,
+            )
           }
         }
       }

data/lib/kafka/fetched_batch.rb

@@ -0,0 +1,16 @@
+module Kafka
+  class FetchedBatch
+    attr_reader :topic, :partition, :highwater_mark_offset, :messages
+
+    def initialize(topic:, partition:, highwater_mark_offset:, messages:)
+      @topic = topic
+      @partition = partition
+      @highwater_mark_offset = highwater_mark_offset
+      @messages = messages
+    end
+
+    def empty?
+      @messages.empty?
+    end
+  end
+end

data/lib/kafka/message_buffer.rb

@@ -14,8 +14,8 @@ module Kafka
       @bytesize = 0
     end
 
-    def write(value:, key:, topic:, partition:)
-      message = Protocol::Message.new(key: key, value: value)
+    def write(value:, key:, topic:, partition:, create_time: Time.now)
+      message = Protocol::Message.new(key: key, value: value, create_time: create_time)
 
       buffer_for(topic, partition) << message
 
@@ -60,8 +60,8 @@ module Kafka
       @buffer.delete(topic) if @buffer[topic].empty?
     end
 
-    def message_count_for_partition(topic:, partition:)
-      buffer_for(topic, partition).count
+    def messages_for(topic:, partition:)
+      buffer_for(topic, partition)
     end
 
     # Clears messages across all topics and partitions.

data/lib/kafka/pending_message.rb

@@ -2,14 +2,15 @@ module Kafka
   class PendingMessage
     attr_reader :value, :key, :topic, :partition, :partition_key
 
-    attr_reader :bytesize
+    attr_reader :bytesize, :create_time
 
-    def initialize(value:, key:, topic:, partition:, partition_key:)
+    def initialize(value:, key:, topic:, partition:, partition_key:, create_time:)
       @key = key
       @value = value
       @topic = topic
       @partition = partition
       @partition_key = partition_key
+      @create_time = create_time
 
       @bytesize = key.to_s.bytesize + value.to_s.bytesize
     end

data/lib/kafka/produce_operation.rb

@@ -104,18 +104,21 @@ module Kafka
       response.each_partition do |topic_info, partition_info|
         topic = topic_info.topic
         partition = partition_info.partition
-        offset = partition_info.offset
-        message_count = @buffer.message_count_for_partition(topic: topic, partition: partition)
+        messages = @buffer.messages_for(topic: topic, partition: partition)
+        ack_time = Time.now
 
         begin
           Protocol.handle_error(partition_info.error_code)
 
-          Instrumentation.instrument("ack_messages.producer.kafka", {
-            topic: topic,
-            partition: partition,
-            offset: offset,
-            message_count: message_count,
-          })
+          messages.each do |message|
+            Instrumentation.instrument("ack_message.producer.kafka", {
+              key: message.key,
+              value: message.value,
+              topic: topic,
+              partition: partition,
+              delay: ack_time - message.create_time,
+            })
+          end
         rescue Kafka::CorruptMessage
           @logger.error "Corrupt message when writing to #{topic}/#{partition}"
         rescue Kafka::UnknownTopicOrPartition
@@ -133,7 +136,7 @@ module Kafka
         rescue Kafka::NotEnoughReplicasAfterAppend
           @logger.error "Messages written, but to fewer in-sync replicas than required for #{topic}/#{partition}"
         else
-          @logger.debug "Successfully appended #{message_count} messages to #{topic}/#{partition} at offset #{offset}"
+          @logger.debug "Successfully appended #{messages.count} messages to #{topic}/#{partition}"
 
           # The messages were successfully written; clear them from the buffer.
           @buffer.clear_messages(topic: topic, partition: partition)

data/lib/kafka/producer.rb

@@ -181,12 +181,15 @@ module Kafka
     # @raise [BufferOverflow] if the maximum buffer size has been reached.
     # @return [nil]
     def produce(value, key: nil, topic:, partition: nil, partition_key: nil)
+      create_time = Time.now
+
       message = PendingMessage.new(
         value: value,
         key: key,
         topic: topic,
         partition: partition,
         partition_key: partition_key,
+        create_time: create_time,
       )
 
       if buffer_size >= @max_buffer_size
@@ -204,6 +207,7 @@ module Kafka
         value: value,
         key: key,
         topic: topic,
+        create_time: create_time,
         buffer_size: buffer_size,
         max_buffer_size: @max_buffer_size,
       })
@@ -322,6 +326,7 @@ module Kafka
           key: message.key,
           topic: message.topic,
           partition: partition,
+          create_time: message.create_time,
         )
       end
     rescue Kafka::Error => e

data/lib/kafka/protocol/decoder.rb

@@ -92,7 +92,13 @@ module Kafka
       #
       # @return [String]
       def read(number_of_bytes)
-        @io.read(number_of_bytes) or raise EOFError
+        data = @io.read(number_of_bytes) or raise EOFError
+
+        # If the `read` call returned less data than expected we should not
+        # proceed.
+        raise EOFError if data.size != number_of_bytes
+
+        data
       end
     end
   end

data/lib/kafka/protocol/message.rb

@@ -16,15 +16,16 @@ module Kafka
     class Message
       MAGIC_BYTE = 0
 
-      attr_reader :key, :value, :attributes, :offset
+      attr_reader :key, :value, :codec_id, :offset
 
-      attr_reader :bytesize
+      attr_reader :bytesize, :create_time
 
-      def initialize(value:, key: nil, attributes: 0, offset: -1)
+      def initialize(value:, key: nil, create_time: Time.now, codec_id: 0, offset: -1)
         @key = key
         @value = value
-        @attributes = attributes
+        @codec_id = codec_id
         @offset = offset
+        @create_time = create_time
 
         @bytesize = @key.to_s.bytesize + @value.to_s.bytesize
       end
@@ -39,17 +40,17 @@ module Kafka
       def ==(other)
         @key == other.key &&
           @value == other.value &&
-          @attributes == other.attributes &&
+          @codec_id == other.codec_id &&
           @offset == other.offset
       end
 
       def compressed?
-        @attributes != 0
+        @codec_id != 0
       end
 
       # @return [Kafka::Protocol::MessageSet]
       def decompress
-        codec = Compression.find_codec_by_id(@attributes)
+        codec = Compression.find_codec_by_id(@codec_id)
 
         # For some weird reason we need to cut out the first 20 bytes.
         data = codec.decompress(value)
@@ -73,7 +74,11 @@ module Kafka
         key = message_decoder.bytes
         value = message_decoder.bytes
 
-        new(key: key, value: value, attributes: attributes, offset: offset)
+        # The codec id is encoded in the three least significant bits of the
+        # attributes.
+        codec_id = attributes & 0b111
+
+        new(key: key, value: value, codec_id: codec_id, offset: offset)
       end
 
       private
@@ -96,7 +101,7 @@ module Kafka
         encoder = Encoder.new(buffer)
 
         encoder.write_int8(MAGIC_BYTE)
-        encoder.write_int8(@attributes)
+        encoder.write_int8(@codec_id)
         encoder.write_bytes(@key)
         encoder.write_bytes(@value)
 

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.3.1"
+  VERSION = "0.3.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-03-08 00:00:00.000000000 Z
+date: 2016-03-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -180,6 +180,7 @@ files:
 - lib/kafka/consumer.rb
 - lib/kafka/consumer_group.rb
 - lib/kafka/fetch_operation.rb
+- lib/kafka/fetched_batch.rb
 - lib/kafka/fetched_message.rb
 - lib/kafka/gzip_codec.rb
 - lib/kafka/instrumentation.rb