ruby-kafka 0.5.1 → 0.5.2.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d49d8b32a6bc80b5043338fe73a5bdb06e2d0718
-  data.tar.gz: 0d9aea1954190682562c2f4e2baa2dde3ee562cd
+  metadata.gz: 9d3ed93d12793328cee689f4b0ace7da8e8ba951
+  data.tar.gz: cdf38141d80608ad6c16fa5866be66204b599c7a
 SHA512:
-  metadata.gz: d031ca62f99d9034f1a3671b4710138a76b57ab347adab47062275da7cc8eb303b0b8db34cc502a70384c4dcaa423a968b693d139750b9104da36e30712c3abf
-  data.tar.gz: 15d5145f8fbf7def123156d05976f3f8666089421b59a3f9bff5e82ea2e5a4c629a885b58f4fc59de31c2cb3dd1fa70e3c5cefbdae709a782427692237703508
+  metadata.gz: 23e15780a1c7c9089cc27bac07a077d026ce0e0d4f3132061ecd6da91d42ebea75eb8d8a379c896d2637900c3d355cbedbc306945be08cf9d01a3500289e8722
+  data.tar.gz: 3f336771af6ce3c84175717668e7cd2a7e7405521be2a618e57124592999d153d5fdbb75750081b7013fbb5eacd054ab954ae8255dccc49989aef851a5cebdd8

data/CHANGELOG.md

@@ -4,6 +4,13 @@ Changes and additions to the library will be listed here.
 
 ## Unreleased
 
+- Instrument the start of message/batch processing (#496).
+- Mark `Client#fetch_messages` as stable.
+- Fix the list topics API (#508).
+- Add support for LZ4 compression (#499).
+- Refactor compression codec lookup (#509).
+- Fix compressed message set offset bug (#506).
+
 ## v0.5.1
 
 Requires Kafka 0.10.1+ due to usage of a few new APIs.

data/README.md

@@ -95,7 +95,7 @@ This library is targeting Kafka 0.9 with the v0.4.x series and Kafka 0.10 with t
 
 - **Kafka 0.8:** Full support for the Producer API in ruby-kafka v0.4.x, but no support for consumer groups. Simple message fetching works.
 - **Kafka 0.9:** Full support for the Producer and Consumer API in ruby-kafka v0.4.x.
-- **Kafka 0.10:** Full support for the Producer and Consumer API in ruby-kafka v0.5.x.
+- **Kafka 0.10:** Full support for the Producer and Consumer API in ruby-kafka v0.5.x. Note that you _must_ run version 0.10.1 or higher of Kafka due to limitations in 0.10.0.
 - **Kafka 0.11:** Everything that works with Kafka 0.10 should still work, but so far no features specific to Kafka 0.11 have been added.
 
 This library requires Ruby 2.1 or higher.
@@ -719,6 +719,8 @@ ActiveSupport::Notifications.subscribe(/.*\.kafka$/) do |*args|
 end
 ```
 
+All notification events have the `client_id` key in the payload, referring to the Kafka client id.
+
 #### Producer Notifications
 
 * `produce_message.producer.kafka` is sent whenever a message is produced to a buffer. It includes the following payload:
@@ -735,6 +737,8 @@ end
 
 #### Consumer Notifications
 
+All notifications have `group_id` in the payload, referring to the Kafka consumer group id.
+
 * `process_message.consumer.kafka` is sent whenever a message is processed by a consumer. It includes the following payload:
   * `value` is the message value.
   * `key` is the message key.
@@ -743,12 +747,16 @@ end
   * `offset` is the message's offset within the topic partition. 
   * `offset_lag` is the number of messages within the topic partition that have not yet been consumed.
 
+* `start_process_message.consumer.kafka` is sent before `process_message.consumer.kafka`, and contains the same payload. It is delivered _before_ the message is processed, rather than _after_.
+
 * `process_batch.consumer.kafka` is sent whenever a message batch is processed by a consumer. It includes the following payload:
   * `message_count` is the number of messages in the batch.
   * `topic` is the topic that the message batch was consumed from.
   * `partition` is the topic partition that the message batch was consumed from.
   * `highwater_mark_offset` is the message batch's highest offset within the topic partition. 
-  * `offset_lag` is the number of messages within the topic partition that have not yet been consumed. 
+  * `offset_lag` is the number of messages within the topic partition that have not yet been consumed.
+
+* `start_process_batch.consumer.kafka` is sent before `process_batch.consumer.kafka`, and contains the same payload. It is delivered _before_ the batch is processed, rather than _after_.
 
 * `join_group.consumer.kafka` is sent whenever a consumer joins a consumer group. It includes the following payload:
   * `group_id` is the consumer group id.

data/lib/kafka.rb

@@ -147,15 +147,15 @@ module Kafka
   end
 
   # 29
-  class TopicAuthorizationCode < ProtocolError
+  class TopicAuthorizationFailed < ProtocolError
   end
 
   # 30
-  class GroupAuthorizationCode < ProtocolError
+  class GroupAuthorizationFailed < ProtocolError
   end
 
   # 31
-  class ClusterAuthorizationCode < ProtocolError
+  class ClusterAuthorizationFailed < ProtocolError
   end
 
   # 32

data/lib/kafka/async_producer.rb

@@ -100,7 +100,10 @@ module Kafka
     def produce(value, topic:, **options)
       ensure_threads_running!
 
-      buffer_overflow(topic) if @queue.size >= @max_queue_size
+      if @queue.size >= @max_queue_size
+        buffer_overflow topic,
+          "Cannot produce to #{topic}, max queue size (#{@max_queue_size} messages) reached"
+      end
 
       args = [value, **options.merge(topic: topic)]
       @queue << [:produce, args]
@@ -148,14 +151,12 @@ module Kafka
       @timer_thread ||= Thread.new { @timer.run }
     end
 
-    def buffer_overflow(topic)
+    def buffer_overflow(topic, message)
       @instrumenter.instrument("buffer_overflow.async_producer", {
         topic: topic,
       })
 
-      @logger.error "Cannot produce message to #{topic}, max queue size (#{@max_queue_size}) reached"
-
-      raise BufferOverflow
+      raise BufferOverflow, message
     end
 
     class Timer

data/lib/kafka/broker.rb

@@ -29,10 +29,10 @@ module Kafka
 
     # Fetches cluster metadata from the broker.
     #
-    # @param (see Kafka::Protocol::TopicMetadataRequest#initialize)
+    # @param (see Kafka::Protocol::MetadataRequest#initialize)
     # @return [Kafka::Protocol::MetadataResponse]
     def fetch_metadata(**options)
-      request = Protocol::TopicMetadataRequest.new(**options)
+      request = Protocol::MetadataRequest.new(**options)
 
       send_request(request)
     end

data/lib/kafka/client.rb

@@ -352,8 +352,6 @@ module Kafka
     #
     # See a working example in `examples/simple-consumer.rb`.
     #
-    # @note This API is still alpha level. Don't try to use it in production.
-    #
     # @param topic [String] the topic that messages should be fetched from.
     #
     # @param partition [Integer] the partition that messages should be fetched from.
@@ -451,16 +449,25 @@ module Kafka
       end
     end
 
-    def create_topic(name, **options)
-      @cluster.create_topic(name, **options)
+    # Creates a topic in the cluster.
+    #
+    # @param name [String] the name of the topic.
+    # @param num_partitions [Integer] the number of partitions that should be created
+    #   in the topic.
+    # @param replication_factor [Integer] the replication factor of the topic.
+    # @param timeout [Integer] a duration of time to wait for the topic to be
+    #   completely created.
+    # @raise [Kafka::TopicAlreadyExists] if the topic already exists.
+    # @return [nil]
+    def create_topic(name, num_partitions: 1, replication_factor: 1, timeout: 30)
+      @cluster.create_topic(name, num_partitions: num_partitions, replication_factor: replication_factor, timeout: timeout)
     end
 
     # Lists all topics in the cluster.
     #
     # @return [Array<String>] the list of topic names.
     def topics
-      @cluster.clear_target_topics
-      @cluster.topics
+      @cluster.list_topics
     end
 
     def has_topic?(topic)

data/lib/kafka/cluster.rb

@@ -145,7 +145,7 @@ module Kafka
       raise
     end
 
-    def create_topic(name, num_partitions: 1, replication_factor: 1, timeout: 30)
+    def create_topic(name, num_partitions:, replication_factor:, timeout:)
       options = {
         topics: {
           name => {
@@ -232,6 +232,12 @@ module Kafka
       cluster_info.topics.map(&:topic_name)
     end
 
+    # Lists all topics in the cluster.
+    def list_topics
+      response = random_broker.fetch_metadata(topics: nil)
+      response.topics.map(&:topic_name)
+    end
+
     def disconnect
       @broker_pool.close
     end

data/lib/kafka/compression.rb

@@ -1,23 +1,37 @@
 require "kafka/snappy_codec"
 require "kafka/gzip_codec"
+require "kafka/lz4_codec"
 
 module Kafka
   module Compression
+    CODEC_NAMES = {
+      1 => :gzip,
+      2 => :snappy,
+      3 => :lz4,
+    }.freeze
+
+    CODECS = {
+      :gzip => GzipCodec.new,
+      :snappy => SnappyCodec.new,
+      :lz4 => LZ4Codec.new,
+    }.freeze
+
+    def self.codecs
+      CODECS.keys
+    end
+
     def self.find_codec(name)
-      case name
-      when nil then nil
-      when :snappy then SnappyCodec.new
-      when :gzip then GzipCodec.new
-      else raise "Unknown compression codec #{name}"
+      CODECS.fetch(name) do
+        raise "Unknown compression codec #{name}"
       end
     end
 
     def self.find_codec_by_id(codec_id)
-      case codec_id
-      when 1 then GzipCodec.new
-      when 2 then SnappyCodec.new
-      else raise "Unknown codec id #{codec_id}"
+      codec_name = CODEC_NAMES.fetch(codec_id) do
+        raise "Unknown codec id #{codec_id}"
       end
+
+      find_codec(codec_name)
     end
   end
 end

data/lib/kafka/compressor.rb

@@ -21,14 +21,17 @@ module Kafka
     # @param threshold [Integer] the minimum number of messages in a message set
     #   that will trigger compression.
     def initialize(codec_name: nil, threshold: 1, instrumenter:)
-      @codec = Compression.find_codec(codec_name)
+      # Codec may be nil, in which case we won't compress.
+      @codec = codec_name && Compression.find_codec(codec_name)
+
       @threshold = threshold
       @instrumenter = instrumenter
     end
 
     # @param message_set [Protocol::MessageSet]
+    # @param offset [Integer] used to simulate broker behaviour in tests
     # @return [Protocol::MessageSet]
-    def compress(message_set)
+    def compress(message_set, offset: -1)
       return message_set if @codec.nil? || message_set.size < @threshold
 
       compressed_data = compress_data(message_set)
@@ -36,6 +39,7 @@ module Kafka
       wrapper_message = Protocol::Message.new(
         value: compressed_data,
         codec_id: @codec.codec_id,
+        offset: offset
       )
 
       Protocol::MessageSet.new(messages: [wrapper_message])

data/lib/kafka/connection.rb

@@ -107,7 +107,7 @@ module Kafka
 
         response
       end
-    rescue SystemCallError, EOFError => e
+    rescue SystemCallError, EOFError, IOError => e
       close
 
       raise ConnectionError, "Connection error #{e.class}: #{e}"

data/lib/kafka/consumer.rb

@@ -202,18 +202,31 @@ module Kafka
         )
 
         batches.each do |batch|
+          unless batch.empty?
+            @instrumenter.instrument("fetch_batch.consumer", {
+              topic: batch.topic,
+              partition: batch.partition,
+              offset_lag: batch.offset_lag,
+              highwater_mark_offset: batch.highwater_mark_offset,
+              message_count: batch.messages.count,
+            })
+          end
           batch.messages.each do |message|
-            @instrumenter.instrument("process_message.consumer") do |notification|
-              notification.update(
-                topic: message.topic,
-                partition: message.partition,
-                offset: message.offset,
-                offset_lag: batch.highwater_mark_offset - message.offset - 1,
-                create_time: message.create_time,
-                key: message.key,
-                value: message.value,
-              )
-
+            notification = {
+              topic: message.topic,
+              partition: message.partition,
+              offset: message.offset,
+              offset_lag: batch.highwater_mark_offset - message.offset - 1,
+              create_time: message.create_time,
+              key: message.key,
+              value: message.value,
+            }
+
+            # Instrument an event immediately so that subscribers don't have to wait until
+            # the block is completed.
+            @instrumenter.instrument("start_process_message.consumer", notification)
+
+            @instrumenter.instrument("process_message.consumer", notification) do
               begin
                 yield message
                 @current_offsets[message.topic][message.partition] = message.offset
@@ -278,15 +291,19 @@ module Kafka
 
         batches.each do |batch|
           unless batch.empty?
-            @instrumenter.instrument("process_batch.consumer") do |notification|
-              notification.update(
-                topic: batch.topic,
-                partition: batch.partition,
-                offset_lag: batch.offset_lag,
-                highwater_mark_offset: batch.highwater_mark_offset,
-                message_count: batch.messages.count,
-              )
-
+            notification = {
+              topic: batch.topic,
+              partition: batch.partition,
+              offset_lag: batch.offset_lag,
+              highwater_mark_offset: batch.highwater_mark_offset,
+              message_count: batch.messages.count,
+            }
+
+            # Instrument an event immediately so that subscribers don't have to wait until
+            # the block is completed.
+            @instrumenter.instrument("start_process_batch.consumer", notification)
+
+            @instrumenter.instrument("process_batch.consumer", notification) do
               begin
                 yield batch
                 @current_offsets[batch.topic][batch.partition] = batch.last_offset
@@ -365,6 +382,9 @@ module Kafka
           @logger.error "Leader not available; waiting 1s before retrying"
           @cluster.mark_as_stale!
           sleep 1
+        rescue ConnectionError => e
+          @logger.error "Connection error #{e.class}: #{e.message}"
+          @cluster.mark_as_stale!
         rescue SignalException => e
           @logger.warn "Received signal #{e.message}, shutting down"
           @running = false

data/lib/kafka/datadog.rb

@@ -229,8 +229,12 @@ module Kafka
         # This gets us the write rate.
         increment("producer.produce.messages", tags: tags.merge(topic: topic))
 
+        # Information about typical/average/95p message size.
         histogram("producer.produce.message_size", message_size, tags: tags.merge(topic: topic))
 
+        # Aggregate message size.
+        count("producer.produce.message_size.sum", message_size, tags: tags.merge(topic: topic))
+
         # This gets us the avg/max buffer size per producer.
         histogram("producer.buffer.size", buffer_size, tags: tags)
 

data/lib/kafka/lz4_codec.rb

@@ -0,0 +1,21 @@
+module Kafka
+  class LZ4Codec
+    def initialize
+      require "extlz4"
+    rescue LoadError
+      raise LoadError, "using lz4 compression requires adding a dependency on the `extlz4` gem to your Gemfile."
+    end
+
+    def codec_id
+      3
+    end
+
+    def compress(data)
+      LZ4.encode(data)
+    end
+
+    def decompress(data)
+      LZ4.decode(data)
+    end
+  end
+end

data/lib/kafka/protocol.rb

@@ -71,9 +71,9 @@ module Kafka
       26 => InvalidSessionTimeout,
       27 => RebalanceInProgress,
       28 => InvalidCommitOffsetSize,
-      29 => TopicAuthorizationCode,
-      30 => GroupAuthorizationCode,
-      31 => ClusterAuthorizationCode,
+      29 => TopicAuthorizationFailed,
+      30 => GroupAuthorizationFailed,
+      31 => ClusterAuthorizationFailed,
       32 => InvalidTimestamp,
       33 => UnsupportedSaslMechanism,
       34 => InvalidSaslState,
@@ -113,7 +113,7 @@ module Kafka
   end
 end
 
-require "kafka/protocol/topic_metadata_request"
+require "kafka/protocol/metadata_request"
 require "kafka/protocol/metadata_response"
 require "kafka/protocol/produce_request"
 require "kafka/protocol/produce_response"

data/lib/kafka/protocol/encoder.rb

@@ -74,8 +74,13 @@ module Kafka
       # @param array [Array]
       # @return [nil]
       def write_array(array, &block)
-        write_int32(array.size)
-        array.each(&block)
+        if array.nil?
+          # An array can be null, which is different from it being empty.
+          write_int32(-1)
+        else
+          write_int32(array.size)
+          array.each(&block)
+        end
       end
 
       # Writes a string to the IO object.

data/lib/kafka/protocol/message.rb

@@ -58,18 +58,7 @@ module Kafka
         message_set_decoder = Decoder.from_string(data)
         message_set = MessageSet.decode(message_set_decoder)
 
-        # The contained messages need to have their offset corrected.
-        messages = message_set.messages.each_with_index.map do |message, i|
-          Message.new(
-            offset: offset + i,
-            value: message.value,
-            key: message.key,
-            create_time: message.create_time,
-            codec_id: message.codec_id
-          )
-        end
-
-        MessageSet.new(messages: messages)
+        correct_offsets(message_set)
       end
 
       def self.decode(decoder)
@@ -113,6 +102,36 @@ module Kafka
 
       private
 
+      # Offsets may be relative with regards to wrapped message offset, but there are special cases.
+      #
+      # Cases when client will receive corrected offsets:
+      #   - When fetch request is version 0, kafka will correct relative offset on broker side before replying fetch response
+      #   - When messages is stored in 0.9 format on disk (broker configured to do so).
+      #
+      # All other cases, compressed inner messages should have relative offset, with below attributes:
+      #   - The container message should have the 'real' offset
+      #   - The container message's offset should be the 'real' offset of the last message in the compressed batch
+      def correct_offsets(message_set)
+        max_relative_offset = message_set.messages.last.offset
+
+        # The offsets are already correct, do nothing.
+        return message_set if max_relative_offset == offset
+
+        # The contained messages have relative offsets, and needs to be corrected.
+        base_offset = offset - max_relative_offset
+        messages = message_set.messages.map do |message|
+          Message.new(
+            offset: message.offset + base_offset,
+            value: message.value,
+            key: message.key,
+            create_time: message.create_time,
+            codec_id: message.codec_id
+          )
+        end
+
+        MessageSet.new(messages: messages)
+      end
+
       def encode_with_crc
         buffer = StringIO.new
         encoder = Encoder.new(buffer)

data/lib/kafka/protocol/{topic_metadata_request.rb → metadata_request.rb}

@@ -1,6 +1,6 @@
 module Kafka
   module Protocol
-    class TopicMetadataRequest
+    class MetadataRequest
 
       # A request for cluster metadata.
       #

data/lib/kafka/protocol/metadata_response.rb

@@ -1,7 +1,7 @@
 module Kafka
   module Protocol
 
-    # A response to a {TopicMetadataRequest}.
+    # A response to a {MetadataRequest}.
     #
     # The response contains information on the brokers, topics, and partitions in
     # the cluster.

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.5.1"
+  VERSION = "0.5.2.beta1"
 end

data/ruby-kafka.gemspec

@@ -35,6 +35,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rspec-benchmark"
   spec.add_development_dependency "activesupport"
   spec.add_development_dependency "snappy"
+  spec.add_development_dependency "extlz4"
   spec.add_development_dependency "colored"
   spec.add_development_dependency "rspec_junit_formatter", "0.2.2"
   spec.add_development_dependency "dogstatsd-ruby", ">= 3.0.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.2.beta1
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-11-14 00:00:00.000000000 Z
+date: 2017-12-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -136,6 +136,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: extlz4
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: colored
   requirement: !ruby/object:Gem::Requirement
@@ -298,6 +312,7 @@ files:
 - lib/kafka/gzip_codec.rb
 - lib/kafka/heartbeat.rb
 - lib/kafka/instrumenter.rb
+- lib/kafka/lz4_codec.rb
 - lib/kafka/message_buffer.rb
 - lib/kafka/offset_manager.rb
 - lib/kafka/partitioner.rb
@@ -328,6 +343,7 @@ files:
 - lib/kafka/protocol/member_assignment.rb
 - lib/kafka/protocol/message.rb
 - lib/kafka/protocol/message_set.rb
+- lib/kafka/protocol/metadata_request.rb
 - lib/kafka/protocol/metadata_response.rb
 - lib/kafka/protocol/offset_commit_request.rb
 - lib/kafka/protocol/offset_commit_response.rb
@@ -340,7 +356,6 @@ files:
 - lib/kafka/protocol/sasl_handshake_response.rb
 - lib/kafka/protocol/sync_group_request.rb
 - lib/kafka/protocol/sync_group_response.rb
-- lib/kafka/protocol/topic_metadata_request.rb
 - lib/kafka/round_robin_assignment_strategy.rb
 - lib/kafka/sasl/gssapi.rb
 - lib/kafka/sasl/plain.rb
@@ -382,9 +397,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.6.13