logstash-output-kafka 2.0.4 → 2.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6a35674ec070d791b3ad239648717d2beaa06ec8
-  data.tar.gz: 74e8c4d77a987a8ccc8b8cc5881fdfade757a249
+  metadata.gz: 7b5b8efdd636a8bb4e471b679485d416f945db02
+  data.tar.gz: c01e2b497f395cbbcba1337f27b84bb018ad3aea
 SHA512:
-  metadata.gz: b980f514c78301c9135f0558e5a7eeaae2b0c793b3cc9eaeaed67cf5d99ac90b77ffadfe55009a42c644acecf2f79b8058630e732c68c75b7b8721e58ba2b773
-  data.tar.gz: 159fbe47375d3f53bfb8c87e7cd85e4cf7e264d43319d57fab2d4e518735f39d8cb4d90dc81dabf5da53362f82170b42054af9d6b19a775039a8630741b62e5e
+  metadata.gz: a856a0c4fe61dead3491d53fd10eaae3bd9582efbb77392f4649a01b88e08934e765cb7eb0dfb55486f3e832af7fc5c248770dfcaa5226bc47d4485fd1cca999
+  data.tar.gz: 261084bc6d8b42e46e202e50b7f419515be42206d57415d00b9f9e00ede642c238ec8154c6348a14d0313d1d82646d0d44ad5e9df63cdbadc5996931186f2178

data/lib/logstash/outputs/kafka.rb

@@ -38,7 +38,10 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   config :key_serializer, :validate => :string, :default => 'org.apache.kafka.common.serialization.StringSerializer'
   # Serializer class for the value of the message
   config :value_serializer, :validate => :string, :default => 'org.apache.kafka.common.serialization.StringSerializer'
-  # The key for the message
+  # The key that will be included with the record
+  #
+  # If a `message_key` is present, a partition will be chosen using a hash of the key.
+  # If not present, a partition for the message will be assigned in a round-robin fashion.
   config :message_key, :validate => :string
   # The number of acknowledgments the producer requires the leader to have received
   # before considering a request complete.

data/logstash-output-kafka.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-kafka'
-  s.version         = '2.0.4'
+  s.version         = '2.0.5'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'Output events to a Kafka topic. This uses the Kafka Producer API to write messages to a topic on the broker'
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -24,9 +24,8 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'logstash-codec-json'
 
-  s.add_runtime_dependency 'jruby-kafka', '1.6.0'
+  s.add_runtime_dependency 'jruby-kafka', '1.5.0'
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'poseidon'
-  s.add_development_dependency 'longshoreman'
 end

data/spec/integration/outputs/kafka_spec.rb

@@ -2,55 +2,20 @@
 
 require "logstash/devutils/rspec/spec_helper"
 require 'logstash/outputs/kafka'
-require 'longshoreman'
 require 'jruby-kafka'
 require 'json'
 require 'poseidon'
 
-CONTAINER_NAME = "kafka-#{rand(999).to_s}"
-IMAGE_NAME = 'spotify/kafka'
-IMAGE_TAG = 'latest'
-KAFKA_PORT = 9092
-KAFKA_VERSION = "0.8.2.1"
-KAFKA_SCALA_VERSION = "2.11"
-
-RSpec.configure do |config|
-  config.before(:all, :integration => true) do
-    @ls = begin
-            ls = Longshoreman.new
-            ls.container.get(CONTAINER_NAME)
-            ls
-          rescue Docker::Error::NotFoundError
-            Longshoreman.pull_image(IMAGE_NAME, IMAGE_TAG)
-            Longshoreman.new("#{IMAGE_NAME}:#{IMAGE_TAG}", CONTAINER_NAME,
-                             { "ENV" => [ "ADVERTISED_HOST=#{Longshoreman.new.get_host_ip}", "ADVERTISED_PORT=#{KAFKA_PORT}"],
-                               "PortBindings" => { "#{KAFKA_PORT}/tcp" => [{ "HostPort" => "#{KAFKA_PORT}" }]}})
-          end
-    @kafka_host = @ls.get_host_ip
-    @kafka_port = @ls.container.rport(9092)
-    @zk_port = @ls.container.rport(2181)
-  end
-
-  config.after(:suite) do
-    begin
-      ls = Longshoreman::new
-      ls.container.get(CONTAINER_NAME)
-      ls.cleanup
-    rescue Docker::Error::NotFoundError, Excon::Errors::SocketError
-    end
-  end
-end
-
 describe "outputs/kafka", :integration => true do
   let(:test_topic) { 'test' }
-  let(:base_config) { {'client_id' => 'spectest', 'bootstrap_servers' => "#{@kafka_host}:#{@kafka_port}"} }
+  let(:base_config) { {'client_id' => 'spectest' } }
   let(:event) { LogStash::Event.new({'message' => 'hello', '@timestamp' => LogStash::Timestamp.at(0) }) }
 
 
   context 'when outputting messages' do
     let(:num_events) { 3 }
     let(:consumer) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", @kafka_host, @kafka_port,
+      Poseidon::PartitionConsumer.new("my_test_consumer", "localhost", 9092,
                                       test_topic, 0, :earliest_offset)
     end
     subject do
@@ -77,18 +42,15 @@ describe "outputs/kafka", :integration => true do
     let(:num_events) { 10 }
     let(:test_topic) { 'test2' }
     let!(:consumer0) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", @kafka_host, @kafka_port,
+      Poseidon::PartitionConsumer.new("my_test_consumer2", "localhost", 9092,
                                       test_topic, 0, :earliest_offset)
     end
     let!(:consumer1) do
-      Poseidon::PartitionConsumer.new("my_test_consumer", @kafka_host, @kafka_port,
+      Poseidon::PartitionConsumer.new("my_test_consumer2", "localhost", 9092,
                                       test_topic, 1, :earliest_offset)
     end
 
     before :each do
-      command = ["/opt/kafka_#{KAFKA_SCALA_VERSION}-#{KAFKA_VERSION}/bin/kafka-topics.sh", "--create", "--topic", "#{test_topic}", "--partitions", "2", "--zookeeper", "#{@kafka_host}:#{@zk_port}", "--replication-factor", "1"]
-      @ls.container.raw.exec(command)
-
       config = base_config.merge({"topic_id" => test_topic, "message_key" => "static_key"})
       kafka = LogStash::Outputs::Kafka.new(config)
       kafka.register

metadata

@@ -1,113 +1,99 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-kafka
 version: !ruby/object:Gem::Version
-  version: 2.0.4
+  version: 2.0.5
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-04-19 00:00:00.000000000 Z
+date: 2016-04-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.0'
   name: logstash-core-plugin-api
-  prerelease: false
-  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
-  name: logstash-codec-plain
+        version: '1.0'
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-json
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-json
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 1.6.0
-  name: jruby-kafka
+        version: '0'
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: jruby-kafka
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 1.6.0
-- !ruby/object:Gem::Dependency
+        version: 1.5.0
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
-  name: logstash-devutils
+        version: 1.5.0
   prerelease: false
-  type: :development
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: poseidon
   prerelease: false
   type: :development
+- !ruby/object:Gem::Dependency
+  name: poseidon
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: longshoreman
   prerelease: false
   type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elastic.co
 executables: []
@@ -125,9 +111,6 @@ files:
 - logstash-output-kafka.gemspec
 - spec/integration/outputs/kafka_spec.rb
 - spec/unit/outputs/kafka_spec.rb
-- vendor/jar-dependencies/runtime-jars/kafka-clients-0.9.0.1.jar
-- vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.13.jar
-- vendor/jar-dependencies/runtime-jars/slf4j-noop-1.7.13.jar
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
 licenses:
 - Apache License (2.0)
@@ -150,7 +133,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.5
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Output events to a Kafka topic. This uses the Kafka Producer API to write messages to a topic on the broker