logstash-output-kafka 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 668a28d4675fe9e8dbf1924f1a43d25e604f7d2e
-  data.tar.gz: f8b4ad8e31b86d726d2debf4429b6746326b0e00
+  metadata.gz: a89fc9e7f6f239df9d54c6f8f55c0a5599c9462a
+  data.tar.gz: 6f80b02adcdc337c8a0b97c7e366ec59032da19a
 SHA512:
-  metadata.gz: ce0e79ade2227ddf5bc580ae2be13aaca3768a54bb481a4186f567678e585c9bb873e0467e9b8482774851fb420f865ac7b4a41c06a1186f3750ce734101374d
-  data.tar.gz: e9c8aa0b08def9d8fb02aa2c6a1ce5d980e45941c4afcaf990e23c2431bb23f03313547015a3b83ef4ee8cf733803210480c99b87d8919020dc960eb553f1e04
+  metadata.gz: 599a1317e6771bec820e7c0fba70bb74234a6fcaa6ff53635be250227bf8ddc28cd620a6df6107ebf7ec108771b03366c6d053736b42dc698d654452c1e87d2f
+  data.tar.gz: c14b27da3f8fe5092c07ceb34021e4bdceb8575debc9466d2f50f9091849f1c6847fc3b3f7094ab61f3dff6d23c08437a91d9ed3ae0dd5844e7ed484e5721758

data/DEVELOPER.md

@@ -23,7 +23,7 @@ See http://kafka.apache.org/documentation.html#producerconfigs for details about
             partitioner_class => ... # string (optional) default: "kafka.producer.DefaultPartitioner"
             request_timeout_ms => ... # number (optional) default: 10000
             producer_type => ... # string (optional), one of ["sync", "async"] default => 'sync'
-            key_serializer_class => ... # string (optional) default: nil
+            key_serializer_class => ... # string (optional) default: kafka.serializer.StringEncoder
             message_send_max_retries => ... # number (optional) default: 3
             retry_backoff_ms => ... # number (optional) default: 100
             topic_metadata_refresh_interval_ms => ... # number (optional) default: 600 * 1000
@@ -33,6 +33,7 @@ See http://kafka.apache.org/documentation.html#producerconfigs for details about
             batch_num_messages => ... # number (optional) default: 200
             send_buffer_bytes => ... # number (optional) default: 100 * 1024
             client_id => ... # string (optional) default: ""
+            partition_key_format => ... # string (optional) default: nil, Provides a way to specify a partition key as a string
         }
     }
 

data/lib/logstash/outputs/kafka.rb

@@ -1,6 +1,6 @@
 require 'logstash/namespace'
 require 'logstash/outputs/base'
-require 'logstash-output-kafka_jars'
+require 'jruby-kafka'
 
 # Write events to a Kafka topic. This uses the Kafka Producer API to write messages to a topic on
 # the broker.
@@ -64,7 +64,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   # throughput) but open the possibility of a failure of the client machine dropping unsent data.
   config :producer_type, :validate => %w( sync async ), :default => 'sync'
   # The serializer class for keys (defaults to the same as for messages if nothing is given)
-  config :key_serializer_class, :validate => :string, :default => nil
+  config :key_serializer_class, :validate => :string, :default => 'kafka.serializer.StringEncoder'
   # This property will cause the producer to automatically retry a failed send request. This
   # property specifies the number of retries when such failures occur. Note that setting a
   # non-zero value here can lead to duplicates in the case of network errors that cause a message
@@ -102,11 +102,21 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
   # The client id is a user-specified string sent in each request to help trace calls. It should
   # logically identify the application making the request.
   config :client_id, :validate => :string, :default => ''
+  # Provides a way to specify a partition key as a string. To specify a partition key for
+  # Kafka, configure a format that will produce the key as a string. Defaults
+  # `key_serializer_class` to `kafka.serializer.StringEncoder` to match. For example, to partition
+  # by host:
+  # [source,ruby]
+  #     output {
+  #       kafka {
+  #           partition_key_format => "%{host}"
+  #       }
+  #     }
+  config :partition_key_format, :validate => :string, :default => nil
 
   public
   def register
     LogStash::Logger.setup_log4j(@logger)
-    require 'jruby-kafka'
 
     options = {
         :broker_list => @broker_list,
@@ -135,7 +145,7 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
 
     @codec.on_event do |event, data|
       begin
-        @producer.send_msg(event.sprintf(@topic_id),nil,data)
+        @producer.send_msg(event.sprintf(@topic_id),@partition_key,data)
       rescue LogStash::ShutdownSignal
         @logger.info('Kafka producer got shutdown signal')
       rescue => e
@@ -151,7 +161,9 @@ class LogStash::Outputs::Kafka < LogStash::Outputs::Base
       finished
       return
     end
+    @partition_key = if @partition_key_format.nil? then nil else event.sprintf(@partition_key_format) end
     @codec.encode(event)
+    @partition_key = nil
   end
 
   def teardown

data/logstash-output-kafka.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-kafka'
-  s.version         = '0.1.6'
+  s.version         = '0.1.7'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = 'Output events to a Kafka topic. This uses the Kafka Producer API to write messages to a topic on the broker'
   s.description     = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -19,10 +19,6 @@ Gem::Specification.new do |s|
   # Special flag to let us know this is actually a logstash plugin
   s.metadata = { 'logstash_plugin' => 'true', 'group' => 'output'}
 
-  # Jar dependencies
-  s.requirements << "jar 'org.apache.kafka:kafka_2.9.2', '0.8.1.1'"
-  s.requirements << "jar 'org.slf4j:slf4j-log4j12', '1.7.10'"
-
   # Gem dependencies
   s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
   s.add_runtime_dependency 'logstash-codec-plain'
@@ -39,6 +35,6 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'ruby-maven', '3.1.1.0.8'
   s.add_runtime_dependency "maven-tools", '1.0.7'
 
-  s.add_runtime_dependency 'jruby-kafka', ['>=0.2.1']
+  s.add_runtime_dependency 'jruby-kafka', ['>= 1.1.0', '< 2.0.0']
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/outputs/kafka_spec.rb

@@ -1,13 +1,12 @@
 # encoding: utf-8
 require "logstash/devutils/rspec/spec_helper"
 require 'logstash/outputs/kafka'
-require 'logstash-output-kafka_jars'
 require 'jruby-kafka'
 require 'json'
 
 describe "outputs/kafka" do
   let (:simple_kafka_config) {{'topic_id' => 'test'}}
-  let (:event) { LogStash::Event.new({'message' => 'hello', 'topic_name' => 'my_topic',
+  let (:event) { LogStash::Event.new({'message' => 'hello', 'topic_name' => 'my_topic', 'host' => '172.0.0.1',
                                       '@timestamp' => LogStash::Timestamp.now}) }
 
   context 'when initializing' do
@@ -44,5 +43,15 @@ describe "outputs/kafka" do
       kafka.register
       kafka.receive(event)
     end
+
+    it 'should support Event#sprintf placeholders in partition_key_format' do
+      partition_field = 'host'
+      expect_any_instance_of(Kafka::Producer).to receive(:send_msg)
+        .with(simple_kafka_config['topic_id'], event[partition_field], event.to_hash.to_json)
+      kafka = LogStash::Outputs::Kafka.new({'topic_id' => simple_kafka_config['topic_id'],
+                                            'partition_key_format' => "%{#{partition_field}}"})
+      kafka.register
+      kafka.receive(event)
+    end
   end
 end

metadata

@@ -1,17 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-kafka
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-02-13 00:00:00.000000000 Z
+date: 2015-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: logstash
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
-  name: logstash
-  prerelease: false
-  type: :runtime
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
@@ -30,104 +28,112 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: 2.0.0
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-plain
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-json
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-codec-json
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: jar-dependencies
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
+        version: 0.1.7
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.1.7
-  name: jar-dependencies
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: ruby-maven
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.1.7
-- !ruby/object:Gem::Dependency
+        version: 3.1.1.0.8
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 3.1.1.0.8
-  name: ruby-maven
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: maven-tools
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 3.1.1.0.8
-- !ruby/object:Gem::Dependency
+        version: 1.0.7
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 1.0.7
-  name: maven-tools
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: jruby-kafka
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - '>='
       - !ruby/object:Gem::Version
-        version: 1.0.7
-- !ruby/object:Gem::Dependency
+        version: 1.1.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.2.1
-  name: jruby-kafka
+        version: 1.1.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
-        version: 0.2.1
-- !ruby/object:Gem::Dependency
+        version: '0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-  name: logstash-devutils
   prerelease: false
   type: :development
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: info@elasticsearch.com
 executables: []
@@ -164,9 +170,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   - - '>='
     - !ruby/object:Gem::Version
       version: '0'
-requirements:
-- jar 'org.apache.kafka:kafka_2.9.2', '0.8.1.1'
-- jar 'org.slf4j:slf4j-log4j12', '1.7.10'
+requirements: []
 rubyforge_project:
 rubygems_version: 2.1.9
 signing_key: