logstash-output-kafka 7.0.3 → 7.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a5ccedfbfedaaf72bad1ec4696411dd1859e1922
-  data.tar.gz: c4180be5c8a6e66d49b421ebf46e962948c03c35
+  metadata.gz: 773d049a3eb04a9cfdfef72b6e36c260461de0a9
+  data.tar.gz: a717066ae5f896259fd073ebb74ee2dad270b14b
 SHA512:
-  metadata.gz: 3fe5acfc009c895074de3d3c84fc188c5bbaba6297f5a0b1c2286fbd25da7688f6f56ad28d9f60c5c4d6ac01ba0bafccb4e5bdccc5b960e8d0c934c21a5312b9
-  data.tar.gz: 4916c1b9c9aac493d83e63a5448b3776603ec2f4dbcaca58e37850e8a19a8e8a0acb8de7e68b6c61a6fc5304c8ad2a92b0b6dce1f48fff8e1cf4d3da3e3abdf1
+  metadata.gz: f1986831b306f4744e625980988fc5e3d936c93aaee1e7ee98e99e92f95d79ea7f2e527974a9c2d73aba5fec53ed84426a219ebca761f85c1cc729cf9a7897a4
+  data.tar.gz: 573738c79bbb910037c942809ac71f4f1ea4b28ce5e49f5ec724db758150eba9f5762462d23bebb9f73d901caabf17b4a32cb9a922e00fb6e97c29029bd3aa63

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 7.0.4
+  - Bugfix: Fixed a bug that broke using `org.apache.kafka.common.serialization.ByteArraySerializer` as the `value_serializer`
+
 ## 7.0.3
   - Bugfix: Sends are now retried until successful. Previously, failed transmissions to Kafka
     could have been lost by the KafkaProducer library. Now we verify transmission explicitly.

data/lib/logstash/outputs/kafka.rb

@@ -3,6 +3,8 @@ require 'logstash/outputs/base'
 require 'java'
 require 'logstash-output-kafka_jars.rb'
 
+java_import org.apache.kafka.clients.producer.ProducerRecord
+
 # Write events to a Kafka topic. This uses the Kafka Producer API to write messages to a topic on
 # the broker.
 #
@@ -188,22 +190,20 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
 
 
     @producer = create_producer
-    @codec.on_event do |event, data|
-      begin
-        if @message_key.nil?
-          record = org.apache.kafka.clients.producer.ProducerRecord.new(event.sprintf(@topic_id), data)
-        else
-          record = org.apache.kafka.clients.producer.ProducerRecord.new(event.sprintf(@topic_id), event.sprintf(@message_key), data)
-        end
-        prepare(record)
-      rescue LogStash::ShutdownSignal
-        @logger.debug('Kafka producer got shutdown signal')
-      rescue => e
-        @logger.warn('kafka producer threw exception, restarting',
-                     :exception => e)
+    if value_serializer == 'org.apache.kafka.common.serialization.StringSerializer'
+      @codec.on_event do |event, data|
+        write_to_kafka(event, data)
+      end
+    elsif value_serializer == 'org.apache.kafka.common.serialization.ByteArraySerializer'
+      @codec.on_event do |event, data|
+        write_to_kafka(event, data.to_java_bytes)
       end
+    else
+      raise ConfigurationError, "'value_serializer' only supports org.apache.kafka.common.serialization.ByteArraySerializer and org.apache.kafka.common.serialization.StringSerializer" 
     end
-  end # def register
+  end
+
+  # def register
 
   def prepare(record)
     # This output is threadsafe, so we need to keep a batch per thread.
@@ -293,6 +293,21 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   end
 
   private
+
+  def write_to_kafka(event, serialized_data)
+    if @message_key.nil?
+      record = ProducerRecord.new(event.sprintf(@topic_id), serialized_data)
+    else
+      record = ProducerRecord.new(event.sprintf(@topic_id), event.sprintf(@message_key), serialized_data)
+    end
+    prepare(record)
+  rescue LogStash::ShutdownSignal
+    @logger.debug('Kafka producer got shutdown signal')
+  rescue => e
+    @logger.warn('kafka producer threw exception, restarting',
+                 :exception => e)
+  end
+
   def create_producer
     begin
       props = java.util.Properties.new

data/logstash-output-kafka.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-kafka'
-  s.version         = '7.0.3'
+  s.version         = '7.0.4'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'Output events to a Kafka topic. This uses the Kafka Producer API to write messages to a topic on the broker'
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/integration/outputs/kafka_spec.rb

@@ -13,7 +13,7 @@ describe "outputs/kafka", :integration => true do
   let(:event) { LogStash::Event.new({'message' => '183.60.215.50 - - [11/Sep/2014:22:00:00 +0000] "GET /scripts/netcat-webserver HTTP/1.1" 200 182 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"', '@timestamp' => LogStash::Timestamp.at(0) }) }
 
 
-  context 'when outputting messages' do
+  context 'when outputting messages serialized as String' do
     let(:test_topic) { 'topic1' }
     let(:num_events) { 3 }
     let(:consumer) do
@@ -38,6 +38,36 @@ describe "outputs/kafka", :integration => true do
 
   end
 
+  context 'when outputting messages serialized as Byte Array' do
+    let(:test_topic) { 'topic1b' }
+    let(:num_events) { 3 }
+    let(:consumer) do
+      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
+                                      test_topic, 0, :earliest_offset)
+    end
+    subject do
+      consumer.fetch
+    end
+
+    before :each do
+      config = base_config.merge(
+        {
+          "topic_id" => test_topic,
+          "value_serializer" => 'org.apache.kafka.common.serialization.ByteArraySerializer'
+        }
+      )
+      load_kafka_data(config)
+    end
+
+    it 'should have data integrity' do
+      expect(subject.size).to eq(num_events)
+      subject.each do |m|
+        expect(m.value).to eq(event.to_s)
+      end
+    end
+
+  end
+
   context 'when setting message_key' do
     let(:num_events) { 10 }
     let(:test_topic) { 'topic2' }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-kafka
 version: !ruby/object:Gem::Version
-  version: 7.0.3
+  version: 7.0.4
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-10-09 00:00:00.000000000 Z
+date: 2017-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement