ruby-kafka 0.4.4 → 0.5.0.beta2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3f09ed3cd80d943c7aa595e43c3de6fd1bd0f883
-  data.tar.gz: 5906a940b05edf1ee429c184e03d4f2490b2454a
+  metadata.gz: d91d5937445f7d3e85fd36801ac52d91f19fb192
+  data.tar.gz: b7baea6a96bd70ba6ec9ff4ad42e206c22e53c7e
 SHA512:
-  metadata.gz: e352a9d47347f413ffafe9e89fca1369a6aa2de17da5b65de2adf3ba659784aea6939b127411626f7889941e084d57d4329b047e5aba26f83d31a5f312b3caaf
-  data.tar.gz: c1e97414fa1a314ec8e6f9f63fe992544d50c3dd6c0650ac1c2f76ba5866a0c3ed3457f38d71021f9669e6dcc904498a2f9c6ee63542234e94fd5ff000ff4c27
+  metadata.gz: 9e0dac8db89b3002a49538f0daf34c7df3c821a22eae2d1615d4bbc97c25f2b4312a0f7cab1c0682a96079213ee53e85e64784a01096d2ca252c9726635d9236
+  data.tar.gz: ef97106a320e4c933189a935f42f8d383dc0619176cee8df5f8b30c060adee157de9b3293e522795f2f276fd063140f7e2284db4e1ee3343c6104eb0773b0468

data/CHANGELOG.md

@@ -4,13 +4,16 @@ Changes and additions to the library will be listed here.
 
 ## Unreleased
 
-## v0.4.4
+## v0.5.0
 
-- Include the offset lag in batch consumer metrics (Statsd).
+- Drops support for Kafka 0.9 in favor of Kafka 0.10 (#381)!
+- Handle cases where there are no partitions to fetch from by sleeping a bit (#439).
+- Handle problems with the broker cache (#440).
 
 ## v0.4.3
 
 - Restart the async producer thread automatically after errors.
+- Include the offset lag in batch consumer metrics (Statsd).
 - Make the default `max_wait_time` more sane.
 - Fix issue with cached default offset lookups (#431).
 - Upgrade to Datadog client version 3.

data/circle.yml

@@ -11,7 +11,7 @@ machine:
 dependencies:
   pre:
     - docker -v
-    - docker pull ches/kafka:0.9.0.1
+    - docker pull ches/kafka:0.10.0.0
     - docker pull jplock/zookeeper:3.4.6
 
 test:

data/lib/kafka/consumer.rb

@@ -216,8 +216,6 @@ module Kafka
 
         # We may not have received any messages, but it's still a good idea to
         # commit offsets if we've processed messages in the last set of batches.
-        # This also ensures the offsets are retained if we haven't read any messages
-        # since the offset retention period has elapsed.
         @offset_manager.commit_offsets_if_necessary
       end
     end
@@ -281,12 +279,6 @@ module Kafka
 
           return if !@running
         end
-
-        # We may not have received any messages, but it's still a good idea to
-        # commit offsets if we've processed messages in the last set of batches.
-        # This also ensures the offsets are retained if we haven't read any messages
-        # since the offset retention period has elapsed.
-        @offset_manager.commit_offsets_if_necessary
       end
     end
 

data/lib/kafka/fetch_operation.rb

@@ -94,6 +94,7 @@ module Kafka
                 topic: fetched_topic.name,
                 partition: fetched_partition.partition,
                 offset: message.offset,
+                create_time: message.create_time,
               )
             }
 

data/lib/kafka/fetched_message.rb

@@ -16,12 +16,16 @@ module Kafka
     # @return [Integer] the offset of the message in the partition.
     attr_reader :offset
 
-    def initialize(value:, key:, topic:, partition:, offset:)
+    # @return [Time] the timestamp of the message.
+    attr_reader :create_time
+
+    def initialize(value:, key:, topic:, partition:, offset:, create_time:)
       @value = value
       @key = key
       @topic = topic
       @partition = partition
       @offset = offset
+      @create_time = create_time
     end
   end
 end

data/lib/kafka/protocol/fetch_request.rb

@@ -30,6 +30,10 @@ module Kafka
         FETCH_API
       end
 
+      def api_version
+        2
+      end
+
       def response_class
         Protocol::FetchResponse
       end

data/lib/kafka/protocol/fetch_response.rb

@@ -38,11 +38,14 @@ module Kafka
 
       attr_reader :topics
 
-      def initialize(topics: [])
+      def initialize(topics: [], throttle_time_ms: 0)
         @topics = topics
+        @throttle_time_ms = throttle_time_ms
       end
 
       def self.decode(decoder)
+        throttle_time_ms = decoder.int32
+
         topics = decoder.array do
           topic_name = decoder.string
 
@@ -68,7 +71,7 @@ module Kafka
           )
         end
 
-        new(topics: topics)
+        new(topics: topics, throttle_time_ms: throttle_time_ms)
       end
     end
   end

data/lib/kafka/protocol/message.rb

@@ -6,15 +6,16 @@ module Kafka
 
     # ## API Specification
     #
-    #     Message => Crc MagicByte Attributes Key Value
+    #     Message => Crc MagicByte Attributes Timestamp Key Value
     #         Crc => int32
     #         MagicByte => int8
     #         Attributes => int8
+    #         Timestamp => int64, in ms
     #         Key => bytes
     #         Value => bytes
     #
     class Message
-      MAGIC_BYTE = 0
+      MAGIC_BYTE = 1
 
       attr_reader :key, :value, :codec_id, :offset
 
@@ -71,6 +72,7 @@ module Kafka
         end
 
         attributes = message_decoder.int8
+        timestamp = message_decoder.int64
         key = message_decoder.bytes
         value = message_decoder.bytes
 
@@ -78,7 +80,7 @@ module Kafka
         # attributes.
         codec_id = attributes & 0b111
 
-        new(key: key, value: value, codec_id: codec_id, offset: offset)
+        new(key: key, value: value, codec_id: codec_id, offset: offset, create_time: Time.at(timestamp/1000.0))
       end
 
       private
@@ -102,6 +104,7 @@ module Kafka
 
         encoder.write_int8(MAGIC_BYTE)
         encoder.write_int8(@codec_id)
+        encoder.write_int64((@create_time.to_f*1000).to_i)
         encoder.write_bytes(@key)
         encoder.write_bytes(@value)
 

data/lib/kafka/protocol/produce_request.rb

@@ -40,6 +40,10 @@ module Kafka
         PRODUCE_API
       end
 
+      def api_version
+        2
+      end
+
       def response_class
         requires_acks? ? Protocol::ProduceResponse : nil
       end

data/lib/kafka/protocol/produce_response.rb

@@ -11,19 +11,21 @@ module Kafka
       end
 
       class PartitionInfo
-        attr_reader :partition, :error_code, :offset
+        attr_reader :partition, :error_code, :offset, :timestamp
 
-        def initialize(partition:, error_code:, offset:)
+        def initialize(partition:, error_code:, offset:, timestamp:)
           @partition = partition
           @error_code = error_code
           @offset = offset
+          @timestamp = timestamp
         end
       end
 
-      attr_reader :topics
+      attr_reader :topics, :throttle_time_ms
 
-      def initialize(topics: [])
+      def initialize(topics: [], throttle_time_ms: 0)
         @topics = topics
+        @throttle_time_ms = throttle_time_ms
       end
 
       def each_partition
@@ -43,13 +45,16 @@ module Kafka
               partition: decoder.int32,
               error_code: decoder.int16,
               offset: decoder.int64,
+              timestamp: Time.at(decoder.int64/1000.0),
             )
           end
 
           TopicInfo.new(topic: topic, partitions: partitions)
         end
 
-        new(topics: topics)
+        throttle_time_ms = decoder.int32
+
+        new(topics: topics, throttle_time_ms: throttle_time_ms)
       end
     end
   end

data/lib/kafka/statsd.rb

@@ -97,7 +97,6 @@ module Kafka
       end
 
       def process_batch(event)
-        lag = event.payload.fetch(:offset_lag)
         messages = event.payload.fetch(:message_count)
         client = event.payload.fetch(:client_id)
         group_id = event.payload.fetch(:group_id)
@@ -110,8 +109,6 @@ module Kafka
           timing("consumer.#{client}.#{group_id}.#{topic}.#{partition}.process_batch.latency", event.duration)
           count("consumer.#{client}.#{group_id}.#{topic}.#{partition}.messages", messages)
         end
-
-        gauge("consumer.#{client}.#{group_id}.#{topic}.#{partition}.lag", lag)
       end
 
       def join_group(event)

data/lib/kafka/version.rb

@@ -1,3 +1,3 @@
 module Kafka
-  VERSION = "0.4.4"
+  VERSION = "0.5.0.beta2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.4.4
+  version: 0.5.0.beta2
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-11-06 00:00:00.000000000 Z
+date: 2017-10-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -376,12 +376,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: A client library for the Kafka distributed commit log.