logstash-output-kafka 5.1.10 → 5.1.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1bcc9ef8d79cfeae2d45b85067cd5e918bbb48c9
-  data.tar.gz: 4600a285a1630607f8f0f473587bbc338803d0ed
+  metadata.gz: ab8c64c41a0113d3e77d4d74adac247424d6a7c5
+  data.tar.gz: ea85a76f270acd51915e1a1a7a80da0e83e9e82e
 SHA512:
-  metadata.gz: 0d58a1bf85115b8ed45395ea08b22adbd22e40cb3f645e9633de1d3b95fa6cbce7c106acbc9061ef82924cec53fa4c7ee7222204f4a4886ece75e7ef75e9b1d4
-  data.tar.gz: a65b714676d04380af86949dc60563d45d7182d72db86232bc700e1b10c328ed3ffa105a9ec5b9bbf1b30dfb65d773e08fd7f08c740531af8d354f6b26040a86
+  metadata.gz: 18927151780d0b0d085e5472b9d8d6a07601bd847163a3bd6578b9359227c73344d4f24ef1ed498d756e3fea8594860e3cafd0ddb15a2eb566b14ed3a340f890
+  data.tar.gz: 76f4b59eec073cefafe18db46fae720cdede9a234d39aab6da3acc4e6588921e089e076afc8a2cd4542ba81572322229fd9e9332a66d38ea8420e5f7a49899f4

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 5.1.11
+  - Bugfix: Sends are now retried until successful. Previously, failed transmissions to Kafka 
+    could have been lost by the KafkaProducer library. Now we verify transmission explicitly.
+    This changes the default 'retry' from 0 to retry-forever. It was a bug that we defaulted
+    to a retry count of 0.
+    https://github.com/logstash-plugins/logstash-output-kafka/pull/151
+
+  - Docs: Fix misleading info about the default codec
+
 ## 5.1.10
   - Doc fixes
 

data/lib/logstash/outputs/kafka.rb

@@ -111,9 +111,15 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   # elapses the client will resend the request if necessary or fail the request if
   # retries are exhausted.
   config :request_timeout_ms, :validate => :string
-  # Setting a value greater than zero will cause the client to
-  # resend any record whose send fails with a potentially transient error.
-  config :retries, :validate => :number, :default => 0
+  # The default retry behavior is to retry until successful. To prevent data loss,
+  # the use of this setting is discouraged.
+  #
+  # If you choose to set `retries`, a value greater than zero will cause the
+  # client to only retry a fixed number of times. This will result in data loss
+  # if a transient error outlasts your retry count.
+  #
+  # A value less than zero is a configuration error.
+  config :retries, :validate => :number
   # The amount of time to wait before attempting to retry a failed produce request to a given topic partition.
   config :retry_backoff_ms, :validate => :number, :default => 100
   # The size of the TCP send buffer to use when sending data.
@@ -175,6 +181,17 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
 
   public
   def register
+    @thread_batch_map = Concurrent::Hash.new
+
+    if !@retries.nil? 
+      if @retries < 0
+        raise ConfigurationError, "A negative retry count (#{@retries}) is not valid. Must be a value >= 0"
+      end
+
+      @logger.warn("Kafka output is configured with finite retry. This instructs Logstash to LOSE DATA after a set number of send attempts fails. If you do not want to lose data if Kafka is down, then you must remove the retry setting.", :retries => @retries)
+    end
+
+
     @producer = create_producer
     @codec.on_event do |event, data|
       begin
@@ -183,7 +200,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
         else
           record = org.apache.kafka.clients.producer.ProducerRecord.new(event.sprintf(@topic_id), event.sprintf(@message_key), data)
         end
-        @producer.send(record)
+        prepare(record)
       rescue LogStash::ShutdownSignal
         @logger.debug('Kafka producer got shutdown signal')
       rescue => e
@@ -191,14 +208,89 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
                      :exception => e)
       end
     end
-
   end # def register
 
-  def receive(event)
-    if event == LogStash::SHUTDOWN
-      return
+  def prepare(record)
+    # This output is threadsafe, so we need to keep a batch per thread.
+    @thread_batch_map[Thread.current].add(record)
+  end
+
+  def multi_receive(events)
+    t = Thread.current
+    if !@thread_batch_map.include?(t)
+      @thread_batch_map[t] = java.util.ArrayList.new(events.size)
+    end
+
+    events.each do |event|
+      break if event == LogStash::SHUTDOWN
+      @codec.encode(event)
+    end
+
+    batch = @thread_batch_map[t]
+    if batch.any?
+      retrying_send(batch)
+      batch.clear
     end
-    @codec.encode(event)
+  end
+
+  def retrying_send(batch)
+    remaining = @retries;
+
+    while batch.any?
+      if !remaining.nil?
+        if remaining < 0
+          # TODO(sissel): Offer to DLQ? Then again, if it's a transient fault,
+          # DLQing would make things worse (you dlq data that would be successful
+          # after the fault is repaired)
+          logger.info("Exhausted user-configured retry count when sending to Kafka. Dropping these events.",
+                      :max_retries => @retries, :drop_count => batch.count)
+          break
+        end
+
+        remaining -= 1
+      end
+
+      failures = []
+
+      futures = batch.collect do |record| 
+        begin
+          # send() can throw an exception even before the future is created.
+          @producer.send(record)
+        rescue org.apache.kafka.common.errors.TimeoutException => e
+          failures << record
+          nil
+        rescue org.apache.kafka.common.errors.InterruptException => e
+          failures << record
+          nil
+        rescue org.apache.kafka.common.errors.SerializationException => e
+          # TODO(sissel): Retrying will fail because the data itself has a problem serializing.
+          # TODO(sissel): Let's add DLQ here.
+          failures << record
+          nil
+        end
+      end.compact
+
+      futures.each_with_index do |future, i|
+        begin
+          result = future.get()
+        rescue => e
+          # TODO(sissel): Add metric to count failures, possibly by exception type.
+          logger.debug? && logger.debug("KafkaProducer.send() failed: #{e}", :exception => e);
+          failures << batch[i]
+        end
+      end
+
+      # No failures? Cool. Let's move on.
+      break if failures.empty?
+
+      # Otherwise, retry with any failed transmissions
+      batch = failures
+      delay = 1.0 / @retry_backoff_ms
+      logger.info("Sending batch to Kafka failed. Will retry after a delay.", :batch_size => batch.size,
+                  :failures => failures.size, :sleep => delay);
+      sleep(delay)
+    end
+
   end
 
   def close
@@ -222,8 +314,8 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
       props.put(kafka::MAX_REQUEST_SIZE_CONFIG, max_request_size.to_s)
       props.put(kafka::RECONNECT_BACKOFF_MS_CONFIG, reconnect_backoff_ms) unless reconnect_backoff_ms.nil?
       props.put(kafka::REQUEST_TIMEOUT_MS_CONFIG, request_timeout_ms) unless request_timeout_ms.nil?
-      props.put(kafka::RETRIES_CONFIG, retries.to_s)
-      props.put(kafka::RETRY_BACKOFF_MS_CONFIG, retry_backoff_ms.to_s)
+      props.put(kafka::RETRIES_CONFIG, retries.to_s) unless retries.nil?
+      props.put(kafka::RETRY_BACKOFF_MS_CONFIG, retry_backoff_ms.to_s) 
       props.put(kafka::SEND_BUFFER_CONFIG, send_buffer_bytes.to_s)
       props.put(kafka::VALUE_SERIALIZER_CLASS_CONFIG, value_serializer)
 

data/logstash-output-kafka.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-kafka'
-  s.version         = '5.1.10'
+  s.version         = '5.1.11'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'Output events to a Kafka topic. This uses the Kafka Producer API to write messages to a topic on the broker'
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/integration/outputs/kafka_spec.rb

@@ -157,7 +157,7 @@ describe "outputs/kafka", :integration => true do
   def load_kafka_data(config)
     kafka = LogStash::Outputs::Kafka.new(config)
     kafka.register
-    num_events.times do kafka.receive(event) end
+    kafka.multi_receive(num_events.times.collect { event })
     kafka.close
   end
 

data/spec/unit/outputs/kafka_spec.rb

@@ -25,34 +25,118 @@ describe "outputs/kafka" do
   context 'when outputting messages' do
     it 'should send logstash event to kafka broker' do
       expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-        .with(an_instance_of(org.apache.kafka.clients.producer.ProducerRecord))
+        .with(an_instance_of(org.apache.kafka.clients.producer.ProducerRecord)).and_call_original
       kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
       kafka.register
-      kafka.receive(event)
+      kafka.multi_receive([event])
     end
 
     it 'should support Event#sprintf placeholders in topic_id' do
       topic_field = 'topic_name'
       expect(org.apache.kafka.clients.producer.ProducerRecord).to receive(:new)
-        .with("my_topic", event.to_s)
-      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+        .with("my_topic", event.to_s).and_call_original
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).and_call_original
       kafka = LogStash::Outputs::Kafka.new({'topic_id' => "%{#{topic_field}}"})
       kafka.register
-      kafka.receive(event)
+      kafka.multi_receive([event])
     end
 
     it 'should support field referenced message_keys' do
       expect(org.apache.kafka.clients.producer.ProducerRecord).to receive(:new)
-        .with("test", "172.0.0.1", event.to_s)
-      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+        .with("test", "172.0.0.1", event.to_s).and_call_original
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).and_call_original
       kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge({"message_key" => "%{host}"}))
       kafka.register
-      kafka.receive(event)
+      kafka.multi_receive([event])
     end
     
     it 'should raise config error when truststore location is not set and ssl is enabled' do
-      kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge({"ssl" => "true"}))
+      kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge("security_protocol" => "SSL"))
       expect { kafka.register }.to raise_error(LogStash::ConfigurationError, /ssl_truststore_location must be set when SSL is enabled/)
     end
   end
+  
+  context "when KafkaProducer#send() raises an exception" do
+    let(:failcount) { (rand * 10).to_i }
+    let(:sendcount) { failcount + 1 }
+
+    let(:exception_classes) { [
+      org.apache.kafka.common.errors.TimeoutException,
+      org.apache.kafka.common.errors.InterruptException,
+      org.apache.kafka.common.errors.SerializationException
+    ] }
+
+    before do
+      count = 0
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+        .exactly(sendcount).times
+        .and_wrap_original do |m, *args|
+        if count < failcount # fail 'failcount' times in a row.
+          count += 1
+          # Pick an exception at random
+          raise exception_classes.shuffle.first.new("injected exception for testing")
+        else
+          m.call(*args) # call original
+        end
+      end
+    end
+
+    it "should retry until successful" do
+      kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
+      kafka.register
+      kafka.multi_receive([event])
+    end
+  end
+
+  context "when a send fails" do
+    context "and the default retries behavior is used" do
+      # Fail this many times and then finally succeed.
+      let(:failcount) { (rand * 10).to_i }
+
+      # Expect KafkaProducer.send() to get called again after every failure, plus the successful one.
+      let(:sendcount) { failcount + 1 }
+
+      it "should retry until successful" do
+        count = 0;
+
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+              .exactly(sendcount).times
+              .and_wrap_original do |m, *args|
+          if count < failcount
+            count += 1
+            # inject some failures.
+
+            # Return a custom Future that will raise an exception to simulate a Kafka send() problem.
+            future = java.util.concurrent.FutureTask.new { raise "Failed" }
+            future.run
+            future
+          else
+            m.call(*args)
+          end
+        end
+        kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
+        kafka.register
+        kafka.multi_receive([event])
+      end
+    end
+
+    context "and when retries is set by the user" do
+      let(:retries) { (rand * 10).to_i }
+      let(:max_sends) { retries + 1 }
+
+      it "should give up after retries are exhausted" do
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+              .at_most(max_sends).times
+              .and_wrap_original do |m, *args|
+          # Always fail.
+          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future.run
+          future
+        end
+        kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge("retries" => retries))
+        kafka.register
+        kafka.multi_receive([event])
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-kafka
 version: !ruby/object:Gem::Version
-  version: 5.1.10
+  version: 5.1.11
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-08-21 00:00:00.000000000 Z
+date: 2017-10-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement