logstash-integration-kafka 10.0.0-java

data/spec/integration/outputs/kafka_spec.rb

@@ -0,0 +1,194 @@
+# encoding: utf-8
+
+require "logstash/devutils/rspec/spec_helper"
+require 'logstash/outputs/kafka'
+require 'json'
+require 'poseidon'
+
+describe "outputs/kafka", :integration => true do
+  let(:kafka_host) { 'localhost' }
+  let(:kafka_port) { 9092 }
+  let(:num_events) { 10 }
+  let(:base_config) { {'client_id' => 'kafkaoutputspec'} }
+  let(:event) { LogStash::Event.new({'message' => '183.60.215.50 - - [11/Sep/2014:22:00:00 +0000] "GET /scripts/netcat-webserver HTTP/1.1" 200 182 "-" "Mozilla/5.0 (compatible; EasouSpider; +http://www.easou.com/search/spider.html)"', '@timestamp' => LogStash::Timestamp.at(0) }) }
+
+
+  context 'when outputting messages serialized as String' do
+    let(:test_topic) { 'logstash_integration_topic1' }
+    let(:num_events) { 3 }
+    let(:consumer) do
+      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
+                                      test_topic, 0, :earliest_offset)
+    end
+    subject do
+      consumer.fetch
+    end
+
+    before :each do
+      config = base_config.merge({"topic_id" => test_topic})
+      load_kafka_data(config)
+    end
+
+    it 'should have data integrity' do
+      expect(subject.size).to eq(num_events)
+      subject.each do |m|
+        expect(m.value).to eq(event.to_s)
+      end
+    end
+
+  end
+
+  context 'when outputting messages serialized as Byte Array' do
+    let(:test_topic) { 'topic1b' }
+    let(:num_events) { 3 }
+    let(:consumer) do
+      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
+                                      test_topic, 0, :earliest_offset)
+    end
+    subject do
+      consumer.fetch
+    end
+
+    before :each do
+      config = base_config.merge(
+        {
+          "topic_id" => test_topic,
+          "value_serializer" => 'org.apache.kafka.common.serialization.ByteArraySerializer'
+        }
+      )
+      load_kafka_data(config)
+    end
+
+    it 'should have data integrity' do
+      expect(subject.size).to eq(num_events)
+      subject.each do |m|
+        expect(m.value).to eq(event.to_s)
+      end
+    end
+
+  end
+
+  context 'when setting message_key' do
+    let(:num_events) { 10 }
+    let(:test_topic) { 'logstash_integration_topic2' }
+    let!(:consumer0) do
+      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
+                                      test_topic, 0, :earliest_offset)
+    end
+    let!(:consumer1) do
+      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
+                                      test_topic, 1, :earliest_offset)
+    end
+
+    before :each do
+      config = base_config.merge({"topic_id" => test_topic, "message_key" => "static_key"})
+      load_kafka_data(config)
+    end
+
+    it 'should send all events to one partition' do
+      expect(consumer0.fetch.size == num_events || consumer1.fetch.size == num_events).to be true
+    end
+  end
+
+  context 'when using gzip compression' do
+    let(:test_topic) { 'logstash_integration_gzip_topic' }
+    let!(:consumer) do
+      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
+                                      test_topic, 0, :earliest_offset)
+    end
+    subject do
+      consumer.fetch
+    end
+
+    before :each do
+      config = base_config.merge({"topic_id" => test_topic, "compression_type" => "gzip"})
+      load_kafka_data(config)
+    end
+
+    it 'should have data integrity' do
+      expect(subject.size).to eq(num_events)
+      subject.each do |m|
+        expect(m.value).to eq(event.to_s)
+      end
+    end
+  end
+
+  context 'when using snappy compression' do
+    let(:test_topic) { 'logstash_integration_snappy_topic' }
+    let!(:consumer) do
+      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
+                                      test_topic, 0, :earliest_offset)
+    end
+    subject do
+      consumer.fetch
+    end
+
+    before :each do
+      config = base_config.merge({"topic_id" => test_topic, "compression_type" => "snappy"})
+      load_kafka_data(config)
+    end
+
+    it 'should have data integrity' do
+      expect(subject.size).to eq(num_events)
+      subject.each do |m|
+        expect(m.value).to eq(event.to_s)
+      end
+    end
+  end
+
+  context 'when using LZ4 compression' do
+    let(:test_topic) { 'logstash_integration_lz4_topic' }
+
+    before :each do
+      config = base_config.merge({"topic_id" => test_topic, "compression_type" => "lz4"})
+      load_kafka_data(config)
+    end
+  end
+
+  context 'when using multi partition topic' do
+    let(:num_events) { 10 }
+    let(:test_topic) { 'logstash_integration_topic3' }
+    let!(:consumer0) do
+      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
+                                      test_topic, 0, :earliest_offset)
+    end
+    let!(:consumer1) do
+      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
+                                      test_topic, 1, :earliest_offset)
+    end
+
+    let!(:consumer2) do
+      Poseidon::PartitionConsumer.new("my_test_consumer", kafka_host, kafka_port,
+                                      test_topic, 2, :earliest_offset)
+    end
+
+    before :each do
+      config = base_config.merge({"topic_id" => test_topic})
+      load_kafka_data(config)
+    end
+
+    it 'should distribute events to all partition' do
+      consumer0_records = consumer0.fetch
+      consumer1_records = consumer1.fetch
+      consumer2_records = consumer2.fetch
+
+      expect(consumer0_records.size > 1 &&
+        consumer1_records.size > 1 &&
+          consumer2_records.size > 1).to be true
+
+      all_records = consumer0_records + consumer1_records + consumer2_records
+      expect(all_records.size).to eq(num_events)
+      all_records.each do |m|
+        expect(m.value).to eq(event.to_s)
+      end
+    end
+  end
+
+  def load_kafka_data(config)
+    kafka = LogStash::Outputs::Kafka.new(config)
+    kafka.register
+    kafka.multi_receive(num_events.times.collect { event })
+    kafka.close
+  end
+
+end

data/spec/unit/inputs/kafka_spec.rb

@@ -0,0 +1,39 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/inputs/kafka"
+require "concurrent"
+
+class MockConsumer
+  def initialize
+    @wake = Concurrent::AtomicBoolean.new(false)
+  end
+
+  def subscribe(topics)
+  end
+  
+  def poll(ms)
+    if @wake.value
+      raise org.apache.kafka.common.errors.WakeupException.new
+    else
+      10.times.map do
+        org.apache.kafka.clients.consumer.ConsumerRecord.new("logstash", 0, 0, "key", "value")
+      end
+    end
+  end
+
+  def close
+  end
+
+  def wakeup
+    @wake.make_true
+  end
+end
+
+describe LogStash::Inputs::Kafka do
+  let(:config) { { 'topics' => ['logstash'], 'consumer_threads' => 4 } }
+  subject { LogStash::Inputs::Kafka.new(config) }
+
+  it "should register" do
+    expect {subject.register}.to_not raise_error
+  end
+end

data/spec/unit/outputs/kafka_spec.rb

@@ -0,0 +1,192 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require 'logstash/outputs/kafka'
+require 'json'
+
+describe "outputs/kafka" do
+  let (:simple_kafka_config) {{'topic_id' => 'test'}}
+  let (:event) { LogStash::Event.new({'message' => 'hello', 'topic_name' => 'my_topic', 'host' => '172.0.0.1',
+                                      '@timestamp' => LogStash::Timestamp.now}) }
+
+  context 'when initializing' do
+    it "should register" do
+      output = LogStash::Plugin.lookup("output", "kafka").new(simple_kafka_config)
+      expect {output.register}.to_not raise_error
+    end
+
+    it 'should populate kafka config with default values' do
+      kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
+      insist {kafka.bootstrap_servers} == 'localhost:9092'
+      insist {kafka.topic_id} == 'test'
+      insist {kafka.key_serializer} == 'org.apache.kafka.common.serialization.StringSerializer'
+    end
+  end
+
+  context 'when outputting messages' do
+    it 'should send logstash event to kafka broker' do
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+        .with(an_instance_of(org.apache.kafka.clients.producer.ProducerRecord)).and_call_original
+      kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
+      kafka.register
+      kafka.multi_receive([event])
+    end
+
+    it 'should support Event#sprintf placeholders in topic_id' do
+      topic_field = 'topic_name'
+      expect(org.apache.kafka.clients.producer.ProducerRecord).to receive(:new)
+        .with("my_topic", event.to_s).and_call_original
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).and_call_original
+      kafka = LogStash::Outputs::Kafka.new({'topic_id' => "%{#{topic_field}}"})
+      kafka.register
+      kafka.multi_receive([event])
+    end
+
+    it 'should support field referenced message_keys' do
+      expect(org.apache.kafka.clients.producer.ProducerRecord).to receive(:new)
+        .with("test", "172.0.0.1", event.to_s).and_call_original
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send).and_call_original
+      kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge({"message_key" => "%{host}"}))
+      kafka.register
+      kafka.multi_receive([event])
+    end
+
+    it 'should raise config error when truststore location is not set and ssl is enabled' do
+      kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge("security_protocol" => "SSL"))
+      expect { kafka.register }.to raise_error(LogStash::ConfigurationError, /ssl_truststore_location must be set when SSL is enabled/)
+    end
+  end
+  
+  context "when KafkaProducer#send() raises an exception" do
+    let(:failcount) { (rand * 10).to_i }
+    let(:sendcount) { failcount + 1 }
+
+    let(:exception_classes) { [
+      org.apache.kafka.common.errors.TimeoutException,
+      org.apache.kafka.common.errors.InterruptException,
+      org.apache.kafka.common.errors.SerializationException
+    ] }
+
+    before do
+      count = 0
+      expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+        .exactly(sendcount).times
+        .and_wrap_original do |m, *args|
+        if count < failcount # fail 'failcount' times in a row.
+          count += 1
+          # Pick an exception at random
+          raise exception_classes.shuffle.first.new("injected exception for testing")
+        else
+          m.call(*args) # call original
+        end
+      end
+    end
+
+    it "should retry until successful" do
+      kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
+      kafka.register
+      kafka.multi_receive([event])
+    end
+  end
+
+  context "when a send fails" do
+    context "and the default retries behavior is used" do
+      # Fail this many times and then finally succeed.
+      let(:failcount) { (rand * 10).to_i }
+
+      # Expect KafkaProducer.send() to get called again after every failure, plus the successful one.
+      let(:sendcount) { failcount + 1 }
+
+      it "should retry until successful" do
+        count = 0;
+
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+              .exactly(sendcount).times
+              .and_wrap_original do |m, *args|
+          if count < failcount
+            count += 1
+            # inject some failures.
+
+            # Return a custom Future that will raise an exception to simulate a Kafka send() problem.
+            future = java.util.concurrent.FutureTask.new { raise "Failed" }
+            future.run
+            future
+          else
+            m.call(*args)
+          end
+        end
+        kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
+        kafka.register
+        kafka.multi_receive([event])
+      end
+    end
+
+    context 'when retries is 0' do
+      let(:retries) { 0  }
+      let(:max_sends) { 1 }
+
+      it "should should only send once" do
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+                                                                                       .once
+                                                                                       .and_wrap_original do |m, *args|
+          # Always fail.
+          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future.run
+          future
+        end
+        kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge("retries" => retries))
+        kafka.register
+        kafka.multi_receive([event])
+      end
+
+      it 'should not sleep' do
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+                                                                                       .once
+                                                                                       .and_wrap_original do |m, *args|
+          # Always fail.
+          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future.run
+          future
+        end
+
+        kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge("retries" => retries))
+        expect(kafka).not_to receive(:sleep).with(anything)
+        kafka.register
+        kafka.multi_receive([event])
+      end
+    end
+
+    context "and when retries is set by the user" do
+      let(:retries) { (rand * 10).to_i }
+      let(:max_sends) { retries + 1 }
+
+      it "should give up after retries are exhausted" do
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+              .at_most(max_sends).times
+              .and_wrap_original do |m, *args|
+          # Always fail.
+          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future.run
+          future
+        end
+        kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge("retries" => retries))
+        kafka.register
+        kafka.multi_receive([event])
+      end
+
+      it 'should only sleep retries number of times' do
+        expect_any_instance_of(org.apache.kafka.clients.producer.KafkaProducer).to receive(:send)
+                                                                                       .at_most(max_sends)
+                                                                                       .and_wrap_original do |m, *args|
+          # Always fail.
+          future = java.util.concurrent.FutureTask.new { raise "Failed" }
+          future.run
+          future
+        end
+        kafka = LogStash::Outputs::Kafka.new(simple_kafka_config.merge("retries" => retries))
+        expect(kafka).to receive(:sleep).exactly(retries).times
+        kafka.register
+        kafka.multi_receive([event])
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,228 @@
+--- !ruby/object:Gem::Specification
+name: logstash-integration-kafka
+version: !ruby/object:Gem::Version
+  version: 10.0.0
+platform: java
+authors:
+- Elastic
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2019-10-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.12
+  name: jar-dependencies
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.12
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.5.0
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.5.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-json
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  name: stud
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: rspec-wait
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: poseidon
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: snappy
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program.
+email: info@elastic.co
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- DEVELOPER.md
+- Gemfile
+- LICENSE
+- NOTICE.TXT
+- README.md
+- docs/index.asciidoc
+- docs/input-kafka.asciidoc
+- docs/output-kafka.asciidoc
+- lib/logstash-integration-kafka_jars.rb
+- lib/logstash/inputs/kafka.rb
+- lib/logstash/outputs/kafka.rb
+- logstash-integration-kafka.gemspec
+- spec/integration/inputs/kafka_spec.rb
+- spec/integration/outputs/kafka_spec.rb
+- spec/unit/inputs/kafka_spec.rb
+- spec/unit/outputs/kafka_spec.rb
+- vendor/jar-dependencies/com/github/luben/zstd-jni/1.4.2-1/zstd-jni-1.4.2-1.jar
+- vendor/jar-dependencies/org/apache/kafka/kafka-clients/2.3.0/kafka-clients-2.3.0.jar
+- vendor/jar-dependencies/org/lz4/lz4-java/1.6.0/lz4-java-1.6.0.jar
+- vendor/jar-dependencies/org/slf4j/slf4j-api/1.7.26/slf4j-api-1.7.26.jar
+- vendor/jar-dependencies/org/xerial/snappy/snappy-java/1.1.7.3/snappy-java-1.1.7.3.jar
+homepage: http://www.elastic.co/guide/en/logstash/current/index.html
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: integration
+  integration_plugins: logstash-input-kafka,logstash-output-kafka
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+- vendor/jar-dependencies
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.14
+signing_key:
+specification_version: 4
+summary: Integration with Kafka - input and output plugins
+test_files:
+- spec/integration/inputs/kafka_spec.rb
+- spec/integration/outputs/kafka_spec.rb
+- spec/unit/inputs/kafka_spec.rb
+- spec/unit/outputs/kafka_spec.rb