logstash-integration-kafka 10.1.0-java → 10.5.1-java

data/logstash-integration-kafka.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-integration-kafka'
-  s.version         = '10.1.0'
+  s.version         = '10.5.1'
   s.licenses        = ['Apache-2.0']
   s.summary         = "Integration with Kafka - input and output plugins"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline "+

data/spec/unit/inputs/kafka_spec.rb

@@ -48,4 +48,54 @@ describe LogStash::Inputs::Kafka do
       expect( subject.send(:create_consumer, 'sample_client-0') ).to be kafka_client
     end
   end
+
+  context 'string integer config' do
+    let(:config) { super.merge('session_timeout_ms' => '25000', 'max_poll_interval_ms' => '345000') }
+
+    it "sets integer values" do
+      expect(org.apache.kafka.clients.consumer.KafkaConsumer).
+          to receive(:new).with(hash_including('session.timeout.ms' => '25000', 'max.poll.interval.ms' => '345000')).
+              and_return kafka_client = double('kafka-consumer')
+
+      expect( subject.send(:create_consumer, 'sample_client-1') ).to be kafka_client
+    end
+  end
+
+  context 'integer config' do
+    let(:config) { super.merge('session_timeout_ms' => 25200, 'max_poll_interval_ms' => 123_000) }
+
+    it "sets integer values" do
+      expect(org.apache.kafka.clients.consumer.KafkaConsumer).
+          to receive(:new).with(hash_including('session.timeout.ms' => '25200', 'max.poll.interval.ms' => '123000')).
+              and_return kafka_client = double('kafka-consumer')
+
+      expect( subject.send(:create_consumer, 'sample_client-2') ).to be kafka_client
+    end
+  end
+
+  context 'string boolean config' do
+    let(:config) { super.merge('enable_auto_commit' => 'false', 'check_crcs' => 'true') }
+
+    it "sets parameters" do
+      expect(org.apache.kafka.clients.consumer.KafkaConsumer).
+          to receive(:new).with(hash_including('enable.auto.commit' => 'false', 'check.crcs' => 'true')).
+              and_return kafka_client = double('kafka-consumer')
+
+      expect( subject.send(:create_consumer, 'sample_client-3') ).to be kafka_client
+      expect( subject.enable_auto_commit ).to be false
+    end
+  end
+
+  context 'boolean config' do
+    let(:config) { super.merge('enable_auto_commit' => true, 'check_crcs' => false) }
+
+    it "sets parameters" do
+      expect(org.apache.kafka.clients.consumer.KafkaConsumer).
+          to receive(:new).with(hash_including('enable.auto.commit' => 'true', 'check.crcs' => 'false')).
+              and_return kafka_client = double('kafka-consumer')
+
+      expect( subject.send(:create_consumer, 'sample_client-4') ).to be kafka_client
+      expect( subject.enable_auto_commit ).to be true
+    end
+  end
 end

data/spec/unit/outputs/kafka_spec.rb

@@ -56,14 +56,15 @@ describe "outputs/kafka" do
     end
   end
 
-  context "when KafkaProducer#send() raises an exception" do
+  context "when KafkaProducer#send() raises a retriable exception" do
     let(:failcount) { (rand * 10).to_i }
     let(:sendcount) { failcount + 1 }
 
     let(:exception_classes) { [
       org.apache.kafka.common.errors.TimeoutException,
+      org.apache.kafka.common.errors.DisconnectException,
+      org.apache.kafka.common.errors.CoordinatorNotAvailableException,
       org.apache.kafka.common.errors.InterruptException,
-      org.apache.kafka.common.errors.SerializationException
     ] }
 
     before do
@@ -88,6 +89,37 @@ describe "outputs/kafka" do
     end
   end
 
+  context "when KafkaProducer#send() raises a non-retriable exception" do
+    let(:failcount) { (rand * 10).to_i }
+
+    let(:exception_classes) { [
+        org.apache.kafka.common.errors.SerializationException,
+        org.apache.kafka.common.errors.RecordTooLargeException,
+        org.apache.kafka.common.errors.InvalidTopicException
+    ] }
+
+    before do
+      count = 0
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+        .exactly(1).times
+        .and_wrap_original do |m, *args|
+        if count < failcount # fail 'failcount' times in a row.
+          count += 1
+          # Pick an exception at random
+          raise exception_classes.shuffle.first.new("injected exception for testing")
+        else
+          m.call(*args) # call original
+        end
+      end
+    end
+
+    it "should not retry" do
+      kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
+      kafka.register
+      kafka.multi_receive([event])
+    end
+  end
+
   context "when a send fails" do
     context "and the default retries behavior is used" do
       # Fail this many times and then finally succeed.
@@ -107,7 +139,7 @@ describe "outputs/kafka" do
             # inject some failures.
 
             # Return a custom Future that will raise an exception to simulate a Kafka send() problem.
-            future = java.util.concurrent.FutureTask.new { raise "Failed" }
+            future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
             future.run
             future
           else
@@ -129,7 +161,7 @@ describe "outputs/kafka" do
                                                                                        .once
                                                                                        .and_wrap_original do |m, *args|
           # Always fail.
-          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run
           future
         end
@@ -143,7 +175,7 @@ describe "outputs/kafka" do
                                                                                        .once
                                                                                        .and_wrap_original do |m, *args|
           # Always fail.
-          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run
           future
         end
@@ -164,7 +196,7 @@ describe "outputs/kafka" do
               .at_most(max_sends).times
               .and_wrap_original do |m, *args|
           # Always fail.
-          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run
           future
         end
@@ -175,10 +207,10 @@ describe "outputs/kafka" do
 
       it 'should only sleep retries number of times' do
         expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
-                                                                                       .at_most(max_sends)
+                                                                                       .at_most(max_sends).times
                                                                                        .and_wrap_original do |m, *args|
           # Always fail.
-          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future = java.util.concurrent.FutureTask.new { raise org.apache.kafka.common.errors.TimeoutException.new("Failed") }
           future.run
           future
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-integration-kafka
 version: !ruby/object:Gem::Version
-  version: 10.1.0
+  version: 10.5.1
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-03-26 00:00:00.000000000 Z
+date: 2020-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement