logstash-output-kafka 7.0.1 → 7.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5a612af7a54364ed111f32cb4917e1628d97911b
-  data.tar.gz: b5f48497dbb62d9523ebe196ffd25b12f9065fff
+  metadata.gz: a5ccedfbfedaaf72bad1ec4696411dd1859e1922
+  data.tar.gz: c4180be5c8a6e66d49b421ebf46e962948c03c35
 SHA512:
-  metadata.gz: 89ac63b16b8ea24fefcfdf37135585c980095fa032e3b6fe093fc86bb4576514e89765712aafe668692820c5bf67458b72ec807e603e517f99617e7cab07e688
-  data.tar.gz: 03806af2fac8d5fa49d8916cdcba45ebb1f9f00a4c8d91b5aa5bca8eaa4ac33d2c303cb3fdf372fdbb040bdb9049f5434b65ba8a93e691355ffd46daceff3a26
+  metadata.gz: 3fe5acfc009c895074de3d3c84fc188c5bbaba6297f5a0b1c2286fbd25da7688f6f56ad28d9f60c5c4d6ac01ba0bafccb4e5bdccc5b960e8d0c934c21a5312b9
+  data.tar.gz: 4916c1b9c9aac493d83e63a5448b3776603ec2f4dbcaca58e37850e8a19a8e8a0acb8de7e68b6c61a6fc5304c8ad2a92b0b6dce1f48fff8e1cf4d3da3e3abdf1

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 7.0.3
+  - Bugfix: Sends are now retried until successful. Previously, failed transmissions to Kafka
+    could have been lost by the KafkaProducer library. Now we verify transmission explicitly.
+    This changes the default 'retry' from 0 to retry-forever. It was a bug that we defaulted
+    to a retry count of 0.
+    https://github.com/logstash-plugins/logstash-output-kafka/pull/151
+
+## 7.0.2
+  - Docs: Fix misleading info about the default codec
+
 ## 7.0.1
   - Fix some documentation issues
 

data/docs/index.asciidoc

@@ -48,19 +48,19 @@ This output supports connecting to Kafka over:
 
 By default security is disabled but can be turned on as needed.
 
-The only required configuration is the topic_id. The default codec is plain,
-so events will be persisted on the broker in plain format. Logstash will encode your messages with not 
-only the message but also with a timestamp and hostname. If you do not want anything but your message 
-passing through, you should make the output configuration something like:
+The only required configuration is the topic_id. 
+
+The default codec is plain. Logstash will encode your events with not only the message field but also with a timestamp and hostname.
+
+If you want the full content of your events to be sent as json, you should set the codec in the output configuration like this:
 [source,ruby]
     output {
       kafka {
-        codec => plain {
-           format => "%{message}"
-        }
+        codec => json
         topic_id => "mytopic"
       }
     }
+    
 For more information see http://kafka.apache.org/documentation.html#theproducer
 
 Kafka producer configuration: http://kafka.apache.org/documentation.html#newproducerconfigs
@@ -291,10 +291,17 @@ retries are exhausted.
 ===== `retries` 
 
   * Value type is <<number,number>>
-  * Default value is `0`
+  * There is no default value for this setting.
+
+The default retry behavior is to retry until successful. To prevent data loss,
+the use of this setting is discouraged.
+
+If you choose to set `retries`, a value greater than zero will cause the
+client to only retry a fixed number of times. This will result in data loss
+if a transport fault exists for longer than your retry count (network outage,
+Kafka down, etc).
 
-Setting a value greater than zero will cause the client to
-resend any record whose send fails with a potentially transient error.
+A value less than zero is a configuration error.
 
 [id="plugins-{type}s-{plugin}-retry_backoff_ms"]
 ===== `retry_backoff_ms` 

data/lib/logstash/outputs/kafka.rb

@@ -109,9 +109,15 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   # elapses the client will resend the request if necessary or fail the request if
   # retries are exhausted.
   config :request_timeout_ms, :validate => :string
-  # Setting a value greater than zero will cause the client to
-  # resend any record whose send fails with a potentially transient error.
-  config :retries, :validate => :number, :default => 0
+  # The default retry behavior is to retry until successful. To prevent data loss,
+  # the use of this setting is discouraged.
+  #
+  # If you choose to set `retries`, a value greater than zero will cause the
+  # client to only retry a fixed number of times. This will result in data loss
+  # if a transient error outlasts your retry count.
+  #
+  # A value less than zero is a configuration error.
+  config :retries, :validate => :number
   # The amount of time to wait before attempting to retry a failed produce request to a given topic partition.
   config :retry_backoff_ms, :validate => :number, :default => 100
   # The size of the TCP send buffer to use when sending data.
@@ -170,6 +176,17 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
 
   public
   def register
+    @thread_batch_map = Concurrent::Hash.new
+
+    if !@retries.nil? 
+      if @retries < 0
+        raise ConfigurationError, "A negative retry count (#{@retries}) is not valid. Must be a value >= 0"
+      end
+
+      @logger.warn("Kafka output is configured with finite retry. This instructs Logstash to LOSE DATA after a set number of send attempts fails. If you do not want to lose data if Kafka is down, then you must remove the retry setting.", :retries => @retries)
+    end
+
+
     @producer = create_producer
     @codec.on_event do |event, data|
       begin
@@ -178,7 +195,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
         else
           record = org.apache.kafka.clients.producer.ProducerRecord.new(event.sprintf(@topic_id), event.sprintf(@message_key), data)
         end
-        @producer.send(record)
+        prepare(record)
       rescue LogStash::ShutdownSignal
         @logger.debug('Kafka producer got shutdown signal')
       rescue => e
@@ -186,14 +203,89 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
                      :exception => e)
       end
     end
-
   end # def register
 
-  def receive(event)
-    if event == LogStash::SHUTDOWN
-      return
+  def prepare(record)
+    # This output is threadsafe, so we need to keep a batch per thread.
+    @thread_batch_map[Thread.current].add(record)
+  end
+
+  def multi_receive(events)
+    t = Thread.current
+    if !@thread_batch_map.include?(t)
+      @thread_batch_map[t] = java.util.ArrayList.new(events.size)
+    end
+
+    events.each do |event|
+      break if event == LogStash::SHUTDOWN
+      @codec.encode(event)
+    end
+
+    batch = @thread_batch_map[t]
+    if batch.any?
+      retrying_send(batch)
+      batch.clear
     end
-    @codec.encode(event)
+  end
+
+  def retrying_send(batch)
+    remaining = @retries;
+
+    while batch.any?
+      if !remaining.nil?
+        if remaining < 0
+          # TODO(sissel): Offer to DLQ? Then again, if it's a transient fault,
+          # DLQing would make things worse (you dlq data that would be successful
+          # after the fault is repaired)
+          logger.info("Exhausted user-configured retry count when sending to Kafka. Dropping these events.",
+                      :max_retries => @retries, :drop_count => batch.count)
+          break
+        end
+
+        remaining -= 1
+      end
+
+      failures = []
+
+      futures = batch.collect do |record| 
+        begin
+          # send() can throw an exception even before the future is created.
+          @producer.send(record)
+        rescue org.apache.kafka.common.errors.TimeoutException => e
+          failures << record
+          nil
+        rescue org.apache.kafka.common.errors.InterruptException => e
+          failures << record
+          nil
+        rescue org.apache.kafka.common.errors.SerializationException => e
+          # TODO(sissel): Retrying will fail because the data itself has a problem serializing.
+          # TODO(sissel): Let's add DLQ here.
+          failures << record
+          nil
+        end
+      end.compact
+
+      futures.each_with_index do |future, i|
+        begin
+          result = future.get()
+        rescue => e
+          # TODO(sissel): Add metric to count failures, possibly by exception type.
+          logger.debug? && logger.debug("KafkaProducer.send() failed: #{e}", :exception => e);
+          failures << batch[i]
+        end
+      end
+
+      # No failures? Cool. Let's move on.
+      break if failures.empty?
+
+      # Otherwise, retry with any failed transmissions
+      batch = failures
+      delay = 1.0 / @retry_backoff_ms
+      logger.info("Sending batch to Kafka failed. Will retry after a delay.", :batch_size => batch.size,
+                  :failures => failures.size, :sleep => delay);
+      sleep(delay)
+    end
+
   end
 
   def close
@@ -217,8 +309,8 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
       props.put(kafka::MAX_REQUEST_SIZE_CONFIG, max_request_size.to_s)
       props.put(kafka::RECONNECT_BACKOFF_MS_CONFIG, reconnect_backoff_ms) unless reconnect_backoff_ms.nil?
       props.put(kafka::REQUEST_TIMEOUT_MS_CONFIG, request_timeout_ms) unless request_timeout_ms.nil?
-      props.put(kafka::RETRIES_CONFIG, retries.to_s)
-      props.put(kafka::RETRY_BACKOFF_MS_CONFIG, retry_backoff_ms.to_s)
+      props.put(kafka::RETRIES_CONFIG, retries.to_s) unless retries.nil?
+      props.put(kafka::RETRY_BACKOFF_MS_CONFIG, retry_backoff_ms.to_s) 
       props.put(kafka::SEND_BUFFER_CONFIG, send_buffer_bytes.to_s)
       props.put(kafka::VALUE_SERIALIZER_CLASS_CONFIG, value_serializer)
 

data/logstash-output-kafka.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-kafka'
-  s.version         = '7.0.1'
+  s.version         = '7.0.3'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'Output events to a Kafka topic. This uses the Kafka Producer API to write messages to a topic on the broker'
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/integration/outputs/kafka_spec.rb

@@ -157,7 +157,7 @@ describe "outputs/kafka", :integration => true do
   def load_kafka_data(config)
     kafka = LogStash::Outputs::Kafka.new(config)
     kafka.register
-    num_events.times do kafka.receive(event) end
+    kafka.multi_receive(num_events.times.collect { event })
     kafka.close
   end
 

data/spec/unit/outputs/kafka_spec.rb

@@ -25,34 +25,118 @@ describe "outputs/kafka" do
   context 'when outputting messages' do
     it 'should send logstash event to kafka broker' do
       expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-        .with(an_instance_of(org.apache.kafka.clients.producer.ProducerRecord))
+        .with(an_instance_of(org.apache.kafka.clients.producer.ProducerRecord)).and_call_original
       kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
       kafka.register
-      kafka.receive(event)
+      kafka.multi_receive([event])
     end
 
     it 'should support Event#sprintf placeholders in topic_id' do
       topic_field = 'topic_name'
       expect(org.apache.kafka.clients.producer.ProducerRecord).to receive(:new)
-        .with("my_topic", event.to_s)
-      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+        .with("my_topic", event.to_s).and_call_original
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).and_call_original
       kafka = LogStash::Outputs::Kafka.new({'topic_id' => "%{#{topic_field}}"})
       kafka.register
-      kafka.receive(event)
+      kafka.multi_receive([event])
     end
 
     it 'should support field referenced message_keys' do
       expect(org.apache.kafka.clients.producer.ProducerRecord).to receive(:new)
-        .with("test", "172.0.0.1", event.to_s)
-      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+        .with("test", "172.0.0.1", event.to_s).and_call_original
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).and_call_original
       kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge({"message_key" => "%{host}"}))
       kafka.register
-      kafka.receive(event)
+      kafka.multi_receive([event])
     end
     
     it 'should raise config error when truststore location is not set and ssl is enabled' do
-      kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge({"ssl" => "true"}))
+      kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge("security_protocol" => "SSL"))
       expect { kafka.register }.to raise_error(LogStash::ConfigurationError, /ssl_truststore_location must be set when SSL is enabled/)
     end
   end
+  
+  context "when KafkaProducer#send() raises an exception" do
+    let(:failcount) { (rand * 10).to_i }
+    let(:sendcount) { failcount + 1 }
+
+    let(:exception_classes) { [
+      org.apache.kafka.common.errors.TimeoutException,
+      org.apache.kafka.common.errors.InterruptException,
+      org.apache.kafka.common.errors.SerializationException
+    ] }
+
+    before do
+      count = 0
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+        .exactly(sendcount).times
+        .and_wrap_original do |m, *args|
+        if count < failcount # fail 'failcount' times in a row.
+          count += 1
+          # Pick an exception at random
+          raise exception_classes.shuffle.first.new("injected exception for testing")
+        else
+          m.call(*args) # call original
+        end
+      end
+    end
+
+    it "should retry until successful" do
+      kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
+      kafka.register
+      kafka.multi_receive([event])
+    end
+  end
+
+  context "when a send fails" do
+    context "and the default retries behavior is used" do
+      # Fail this many times and then finally succeed.
+      let(:failcount) { (rand * 10).to_i }
+
+      # Expect KafkaProducer.send() to get called again after every failure, plus the successful one.
+      let(:sendcount) { failcount + 1 }
+
+      it "should retry until successful" do
+        count = 0;
+
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+              .exactly(sendcount).times
+              .and_wrap_original do |m, *args|
+          if count < failcount
+            count += 1
+            # inject some failures.
+
+            # Return a custom Future that will raise an exception to simulate a Kafka send() problem.
+            future = java.util.concurrent.FutureTask.new { raise "Failed" }
+            future.run
+            future
+          else
+            m.call(*args)
+          end
+        end
+        kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
+        kafka.register
+        kafka.multi_receive([event])
+      end
+    end
+
+    context "and when retries is set by the user" do
+      let(:retries) { (rand * 10).to_i }
+      let(:max_sends) { retries + 1 }
+
+      it "should give up after retries are exhausted" do
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+              .at_most(max_sends).times
+              .and_wrap_original do |m, *args|
+          # Always fail.
+          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future.run
+          future
+        end
+        kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge("retries" => retries))
+        kafka.register
+        kafka.multi_receive([event])
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-kafka
 version: !ruby/object:Gem::Version
-  version: 7.0.1
+  version: 7.0.3
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-08-16 00:00:00.000000000 Z
+date: 2017-10-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement