logstash-kafka 0.8.1-java → 0.8.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1e358d058f86eef4b5fc6fc9f270972593722a11
-  data.tar.gz: ea695e3d5437794f85b30bd27eb0b6ea049360bf
+  metadata.gz: 2e632483abd563894f4eb6aa602d38ac66aa5371
+  data.tar.gz: b17b598b5845a8007ecd45cbf9feb704692b9d1b
 SHA512:
-  metadata.gz: 36e0046ed5591ca0664fbb0b68cefe7b7cee969b1b2ce16894a3f00d30205c79dabc788aa45bc7c45e5b6f95e2f9bebc6a781c25fd89c14597bba74d960b2058
-  data.tar.gz: b87e7e46523bbb18df4a42a1183dfc066145f0a67925b094a74e69d692b758e579e31f62778e0f3894ecddd0076d8a06fc015ce9eb8c8ca760a9f44d33a21708
+  metadata.gz: 62ad722dda07f70aea2aae515da1789f171547a5f87ff3e8d87697aae2d08749fe2fd41c0671a1399cd829a5c069091c15c1cf01377a88363cf26b946a68608e
+  data.tar.gz: 2eb29abf8aa79a5d3fd90b1393b72a970d105184793560079b4d556eb54caaeda796b6fd67ece6eddad45036f466a61100b75b86057a64bffe7b4e6caa84f9f3

data/spec/inputs/kafka_spec.rb

@@ -0,0 +1,101 @@
+require "test_utils"
+require "logstash/inputs/kafka"
+require 'jruby-kafka'
+
+class LogStash::Inputs::TestKafka < LogStash::Inputs::Kafka
+  private
+  def queue_event(msg, output_queue)
+    super(msg, output_queue)
+    # need to raise exception here to stop the infinite loop
+    raise LogStash::ShutdownSignal
+  end
+end
+
+class TestMessageAndMetadata
+  attr_reader :topic, :partition, :key, :message
+  def initialize(topic, partition, key, message)
+    @topic = topic
+    @partition = partition
+    @key = key
+    @message = message
+  end
+end
+
+
+class TestKafkaGroup < Kafka::Group
+  def run(a_num_threads, a_queue)
+    blah = TestMessageAndMetadata.new(@topic, 0, nil, 'Kafka message')
+    a_queue << blah
+  end
+end
+
+describe 'inputs/kafka' do
+  let (:kafka_config) {{'topic_id' => 'test'}}
+  let (:empty_config) {{}}
+  let (:bad_kafka_config) {{'topic_id' => 'test', 'white_list' => 'other_topic'}}
+  let (:white_list_kafka_config) {{'white_list' => 'other_topic'}}
+  let (:decorated_kafka_config) {{'topic_id' => 'test', 'decorate_events' => true}}
+
+  it "should register" do
+    input = LogStash::Plugin.lookup("input", "kafka").new(kafka_config)
+    expect {input.register}.to_not raise_error
+  end
+
+  it "should register with whitelist" do
+    input = LogStash::Plugin.lookup("input", "kafka").new(white_list_kafka_config)
+    expect {input.register}.to_not raise_error
+  end
+
+  it "should fail with multiple topic configs" do
+    input = LogStash::Plugin.lookup("input", "kafka").new(empty_config)
+    expect {input.register}.to raise_error
+  end
+
+  it "should fail without topic configs" do
+    input = LogStash::Plugin.lookup("input", "kafka").new(bad_kafka_config)
+    expect {input.register}.to raise_error
+  end
+
+  it 'should populate kafka config with default values' do
+    kafka = LogStash::Inputs::TestKafka.new(kafka_config)
+    insist {kafka.zk_connect} == 'localhost:2181'
+    insist {kafka.topic_id} == 'test'
+    insist {kafka.group_id} == 'logstash'
+    !insist { kafka.reset_beginning }
+  end
+
+  it 'should retrieve event from kafka' do
+    kafka = LogStash::Inputs::TestKafka.new(kafka_config)
+    expect(kafka).to receive(:create_consumer_group) do |options|
+      TestKafkaGroup.new(options)
+    end
+    kafka.register
+
+    logstash_queue = Queue.new
+    kafka.run logstash_queue
+    e = logstash_queue.pop
+    insist { e['message'] } == 'Kafka message'
+    # no metadata by default
+    insist { e['kafka'] } == nil
+  end
+
+  it 'should retrieve a decorated event from kafka' do
+    kafka = LogStash::Inputs::TestKafka.new(decorated_kafka_config)
+    expect(kafka).to receive(:create_consumer_group) do |options|
+      TestKafkaGroup.new(options)
+    end
+    kafka.register
+
+    logstash_queue = Queue.new
+    kafka.run logstash_queue
+    e = logstash_queue.pop
+    insist { e['message'] } == 'Kafka message'
+    # no metadata by default
+    insist { e['kafka']['topic'] } == 'test'
+    insist { e['kafka']['consumer_group'] } == 'logstash'
+    insist { e['kafka']['msg_size'] } == 13
+    insist { e['kafka']['partition'] } == 0
+    insist { e['kafka']['key'] } == nil
+  end
+
+end

data/spec/outputs/kafka_spec.rb

@@ -0,0 +1,37 @@
+require "test_utils"
+require 'logstash/outputs/kafka'
+require 'jruby-kafka'
+require 'json'
+
+describe "outputs/kafka" do
+  let (:simple_kafka_config) {{'topic_id' => 'test'}}
+  let (:event) { LogStash::Event.new({'message' => 'hello', 'topic_name' => 'my_topic', 'host' => '172.0.0.1',
+                                      '@timestamp' => "2011-08-18T13:00:14.000Z"}) }
+
+  context 'when initializing' do
+    it "should register" do
+      output = LogStash::Plugin.lookup("output", "kafka").new(simple_kafka_config)
+      expect {output.register}.to_not raise_error
+    end
+
+    it 'should populate kafka config with default values' do
+      kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
+      insist {kafka.broker_list} == 'localhost:9092'
+      insist {kafka.topic_id} == 'test'
+      insist {kafka.compression_codec} == 'none'
+      insist {kafka.serializer_class} == 'kafka.serializer.StringEncoder'
+      insist {kafka.partitioner_class} == 'kafka.producer.DefaultPartitioner'
+      insist {kafka.producer_type} == 'sync'
+    end
+  end
+
+  context 'when outputting messages' do
+    it 'should send logstash event to kafka broker' do
+      expect_any_instance_of(Kafka::Producer).to receive(:send_msg)
+                                                     .with(simple_kafka_config['topic_id'], nil, event.to_hash.to_json)
+      kafka = LogStash::Outputs::Kafka.new(simple_kafka_config)
+      kafka.register
+      kafka.receive(event)
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-kafka
 version: !ruby/object:Gem::Version
-  version: 0.8.1
+  version: 0.8.2
 platform: java
 authors:
 - Joseph Lawson
@@ -33,6 +33,8 @@ extra_rdoc_files: []
 files:
 - lib/logstash/inputs/kafka.rb
 - lib/logstash/outputs/kafka.rb
+- spec/inputs/kafka_spec.rb
+- spec/outputs/kafka_spec.rb
 homepage: https://github.com/joekiller/logstash-kafka
 licenses:
 - Apache 2.0
@@ -57,4 +59,6 @@ rubygems_version: 2.4.6
 signing_key: 
 specification_version: 4
 summary: Provides input and output plugin functionality for Logstash 1.4.X
-test_files: []
+test_files:
+- spec/inputs/kafka_spec.rb
+- spec/outputs/kafka_spec.rb