ruby-kafka 0.7.0.alpha4 → 0.7.0.beta1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 93553029514a22802156abc20e70cf63263f2ddc6bfe4312e11bbb629eefd017
-  data.tar.gz: 14c14261e50408e8ca0b0c5f14cac94939f6756bc8c5499fd2af85280218dc80
+  metadata.gz: 1662d1f7534da36a4a6f5ade9e194fd5142034ba4b53d1b6291fcf233e7eb684
+  data.tar.gz: 36b9ecb999f55cfd797c23da795c214c7b9712b42a88e6343024e0e17a123eac
 SHA512:
-  metadata.gz: 264f0d854be944a050f7392f43c231a3499d786b5d27bf772a6a2c7b05ed7d496f8c0855e661e4e263667abe9b2d113740d92d842310052e53797588a0375034
-  data.tar.gz: ea31b08620b1451ac663fd16eeab8816948a3a421218282ece5c2b94417931c926ffdcbafb0436e414a3136c9ecab0442a3b477c3787d106f3380c9e18f2fcb0
+  metadata.gz: 8b22b391b89199f8001401069f5040bac248866f82b8bd1f69ecf3d67d947ef7f54f6d933a68a9114c2a28f54097be4cedb3c74743bae1b231f313a4427ad257
+  data.tar.gz: '0318b5a5cd085ea2c96e7e1bf3c91d914c3898d555b6bdf5681851907e0dec4e2f3ab39c95bf8c289558c41d0c901b243ea7e240e2271235769a2378069e1bbc'

data/CHANGELOG.md

@@ -5,6 +5,20 @@ Changes and additions to the library will be listed here.
 ## Unreleased
 
 - Drop support for Kafka 0.10 in favor of native support for Kafka 0.11.
+- Support record headers (#604).
+- Add instrumenter and logger when async message delivery fails (#603).
+
+## 0.6.7
+
+- Handle case where consumer doesn't know about the topic (#597 + 0e302cbd0f31315bf81c1d1645520413ad6b58f0)
+
+## v0.6.5
+
+- Fix bug related to partition assignment.
+
+## v0.6.4
+
+- Fix bug that caused consumers to jump back and reprocess messages (#595).
 
 ## v0.6.3
 

data/lib/kafka/async_producer.rb

@@ -241,8 +241,10 @@ module Kafka
 
       def deliver_messages
         @producer.deliver_messages
-      rescue DeliveryFailed, ConnectionError
-        # Failed to deliver messages -- nothing to do but try again later.
+      rescue DeliveryFailed, ConnectionError => e
+        # Failed to deliver messages -- nothing to do but log and try again later.
+        @logger.error("Failed to asynchronously deliver messages: #{e.message}")
+        @instrumenter.instrument("error.async_producer", { error: e })
       end
 
       def threshold_reached?

data/lib/kafka/client.rb

@@ -109,6 +109,7 @@ module Kafka
     #
     # @param value [String, nil] the message value.
     # @param key [String, nil] the message key.
+    # @param headers [Hash<String, String>] the headers for the message.
     # @param topic [String] the topic that the message should be written to.
     # @param partition [Integer, nil] the partition that the message should be written
     #   to, or `nil` if either `partition_key` is passed or the partition should be
@@ -118,16 +119,17 @@ module Kafka
     # @param retries [Integer] the number of times to retry the delivery before giving
     #   up.
     # @return [nil]
-    def deliver_message(value, key: nil, topic:, partition: nil, partition_key: nil, retries: 1)
+    def deliver_message(value, key: nil, headers: {}, topic:, partition: nil, partition_key: nil, retries: 1)
       create_time = Time.now
 
       message = PendingMessage.new(
-        value,
-        key,
-        topic,
-        partition,
-        partition_key,
-        create_time,
+        value: value,
+        key: key,
+        headers: headers,
+        topic: topic,
+        partition: partition,
+        partition_key: partition_key,
+        create_time: create_time
       )
 
       if partition.nil?
@@ -140,6 +142,7 @@ module Kafka
       buffer.write(
         value: message.value,
         key: message.key,
+        headers: message.headers,
         topic: message.topic,
         partition: partition,
         create_time: message.create_time,

data/lib/kafka/consumer.rb

@@ -37,6 +37,7 @@ module Kafka
   #       puts message.topic
   #       puts message.partition
   #       puts message.key
+  #       puts message.headers
   #       puts message.value
   #       puts message.offset
   #     end
@@ -213,6 +214,7 @@ module Kafka
               create_time: message.create_time,
               key: message.key,
               value: message.value,
+              headers: message.headers
             }
 
             # Instrument an event immediately so that subscribers don't have to wait until

data/lib/kafka/fetched_message.rb

@@ -34,6 +34,11 @@ module Kafka
       @message.create_time
     end
 
+    # @return [Hash<String, String>] the headers of the message.
+    def headers
+      @message.headers
+    end
+
     # @return [Boolean] whether this record is a control record
     def is_control_record
       @message.is_control_record

data/lib/kafka/message_buffer.rb

@@ -16,8 +16,8 @@ module Kafka
       @bytesize = 0
     end
 
-    def write(value:, key:, topic:, partition:, create_time: Time.now)
-      message = Protocol::Record.new(key: key, value: value, create_time: create_time)
+    def write(value:, key:, topic:, partition:, create_time: Time.now, headers: {})
+      message = Protocol::Record.new(key: key, value: value, create_time: create_time, headers: headers)
 
       buffer_for(topic, partition) << message
 

data/lib/kafka/pending_message.rb

@@ -2,11 +2,12 @@
 
 module Kafka
   class PendingMessage
-    attr_reader :value, :key, :topic, :partition, :partition_key, :create_time, :bytesize
+    attr_reader :value, :key, :headers, :topic, :partition, :partition_key, :create_time, :bytesize
 
-    def initialize(value, key, topic, partition, partition_key, create_time)
+    def initialize(value:, key:, headers: {}, topic:, partition:, partition_key:, create_time:)
       @value = value
       @key = key
+      @headers = headers
       @topic = topic
       @partition = partition
       @partition_key = partition_key
@@ -18,6 +19,7 @@ module Kafka
       @value == other.value &&
         @key == other.key &&
         @topic == other.topic &&
+        @headers == other.headers &&
         @partition == other.partition &&
         @partition_key == other.partition_key &&
         @create_time == other.create_time &&

data/lib/kafka/producer.rb

@@ -172,6 +172,7 @@ module Kafka
     #
     # @param value [String] the message data.
     # @param key [String] the message key.
+    # @param headers [Hash<String, String>] the headers for the message.
     # @param topic [String] the topic that the message should be written to.
     # @param partition [Integer] the partition that the message should be written to.
     # @param partition_key [String] the key that should be used to assign a partition.
@@ -179,14 +180,15 @@ module Kafka
     #
     # @raise [BufferOverflow] if the maximum buffer size has been reached.
     # @return [nil]
-    def produce(value, key: nil, topic:, partition: nil, partition_key: nil, create_time: Time.now)
+    def produce(value, key: nil, headers: {}, topic:, partition: nil, partition_key: nil, create_time: Time.now)
       message = PendingMessage.new(
-        value && value.to_s,
-        key && key.to_s,
-        topic.to_s,
-        partition && Integer(partition),
-        partition_key && partition_key.to_s,
-        create_time,
+        value: value && value.to_s,
+        key: key && key.to_s,
+        headers: headers,
+        topic: topic.to_s,
+        partition: partition && Integer(partition),
+        partition_key: partition_key && partition_key.to_s,
+        create_time: create_time
       )
 
       if buffer_size >= @max_buffer_size
@@ -354,6 +356,7 @@ module Kafka
           @buffer.write(
             value: message.value,
             key: message.key,
+            headers: message.headers,
             topic: message.topic,
             partition: partition,
             create_time: message.create_time,
@@ -390,12 +393,13 @@ module Kafka
       @buffer.each do |topic, partition, messages_for_partition|
         messages_for_partition.each do |message|
           messages << PendingMessage.new(
-            message.value,
-            message.key,
-            topic,
-            partition,
-            nil,
-            message.create_time
+            value: message.value,
+            key: message.key,
+            headers: message.headers,
+            topic: topic,
+            partition: partition,
+            partition_key: nil,
+            create_time: message.create_time
           )
         end
       end

data/lib/kafka/protocol/record.rb

@@ -40,8 +40,8 @@ module Kafka
         record_encoder.write_varint_bytes(@value)
 
         record_encoder.write_varint_array(@headers.to_a) do |header_key, header_value|
-          record_encoder.write_varint_string(header_key)
-          record_encoder.write_varint_bytes(header_value)
+          record_encoder.write_varint_string(header_key.to_s)
+          record_encoder.write_varint_bytes(header_value.to_s)
         end
 
         encoder.write_varint_bytes(record_buffer.string)

data/lib/kafka/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kafka
-  VERSION = "0.7.0.alpha4"
+  VERSION = "0.7.0.beta1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ruby-kafka
 version: !ruby/object:Gem::Version
-  version: 0.7.0.alpha4
+  version: 0.7.0.beta1
 platform: ruby
 authors:
 - Daniel Schierbeck
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-06-12 00:00:00.000000000 Z
+date: 2018-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: digest-crc