fluent-plugin-kafka 0.6.3 → 0.6.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 2b9941b6f4988f1fb14b5a0a591a81574411f915
-  data.tar.gz: '08c543d8db1c038e908584460e39a6a9df89e86c'
+SHA256:
+  metadata.gz: efe26ba34b8d77d120aa8c7babfaa69360520698079dee0be5caefb388964e4a
+  data.tar.gz: 0c42b0e54db31526150f1e0f71ee94a7c336112c649e0f39d77a4104f7202268
 SHA512:
-  metadata.gz: 253f7354704fecc1fa1d21bc84056727169fc9dcbcbde55afe595ae9c882213dae910c82bd2b7e34918169f9275f6a8ab031646405409a115912a92f93092b94
-  data.tar.gz: 87cdd5a750bc91c036901d64f76fda7ae874a7fcc16f9f573311bfc5a061492e6d5a3612c6c3378bf7ab34f7a99b8f6b029d75405a2ef240cf3bb40c709498f0
+  metadata.gz: 95baf194bcc5a486cf1c924500468f49f31a6c9f55ce39f811f37b221017f75956002110c260ab1577f8d701b4f917364ce6042d9507a105254ca3c013395ca8
+  data.tar.gz: 8de6a01fd63e12c6f34ab91522db088b6fd25fc6d1b2d924d3b1e69c4fb11f7e773ae07a0be6790a7bfcb7e5c6f060594ab4b1c84c5eb4f856a40b52a3c6cf02

data/ChangeLog

@@ -1,3 +1,10 @@
+Release 0.6.4 - 2017/11/23
+
+	* Multi worker support for v0.14
+	* Add kafka_message_key parameter to input plugins
+	* Relax ruby-kafka version for 0.5 or later
+	* Use oj for json when it is installed
+
 Release 0.6.3 - 2017/11/14
 
 	* in_kafka_group: re-create consumer when error happens during event fetch

data/LICENSE

@@ -1,22 +1,14 @@
-Copyright (c) 2014 htgc
+Copyright (C) 2014 htgc
 
-MIT License
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
 
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
+       http://www.apache.org/licenses/LICENSE-2.0
 
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/fluent-plugin-kafka.gemspec

@@ -6,18 +6,19 @@ Gem::Specification.new do |gem|
   gem.description   = %q{Fluentd plugin for Apache Kafka > 0.8}
   gem.summary       = %q{Fluentd plugin for Apache Kafka > 0.8}
   gem.homepage      = "https://github.com/fluent/fluent-plugin-kafka"
+  gem.license       = "Apache-2.0"
 
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.6.3'
+  gem.version       = '0.6.4'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
   gem.add_dependency 'ltsv'
-  gem.add_dependency 'ruby-kafka', '~> 0.4.1'
+  gem.add_dependency 'ruby-kafka', '>= 0.4.1', '< 1.0.0'
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "test-unit", ">= 3.0.8"
 end

data/lib/fluent/plugin/in_kafka.rb

@@ -36,6 +36,8 @@ class Fluent::KafkaInput < Fluent::Input
                :desc => "Replace message timestamp with contents of 'time' field."
   config_param :time_format, :string, :default => nil,
                :desc => "Time format to be used to parse 'time' filed."
+  config_param :kafka_message_key, :string, :default => nil,
+               :desc => "Set kafka's message key to this field"
 
   # Kafka#fetch_messages options
   config_param :max_bytes, :integer, :default => nil,
@@ -113,12 +115,22 @@ class Fluent::KafkaInput < Fluent::Input
   def setup_parser
     case @format
     when 'json'
-      require 'yajl'
-      Proc.new { |msg, te|
-        r = Yajl::Parser.parse(msg.value)
-        add_offset_in_hash(r, te, msg.offset) if @add_offset_in_record
-        r
-      }
+      begin
+        require 'oj'
+        Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
+        Proc.new { |msg, te|
+          r = Oj.load(msg.value)
+          add_offset_in_hash(r, te, msg.offset) if @add_offset_in_record
+          r
+        }
+      rescue LoadError
+        require 'yajl'
+        Proc.new { |msg, te|
+          r = Yajl::Parser.parse(msg.value)
+          add_offset_in_hash(r, te, msg.offset) if @add_offset_in_record
+          r
+        }
+      end
     when 'ltsv'
       require 'ltsv'
       Proc.new { |msg, te|
@@ -175,6 +187,7 @@ class Fluent::KafkaInput < Fluent::Input
         @add_suffix,
         offset_manager,
         router,
+        @kafka_message_key,
         opt)
     }
     @topic_watchers.each {|tw|
@@ -199,7 +212,7 @@ class Fluent::KafkaInput < Fluent::Input
   end
 
   class TopicWatcher < Coolio::TimerWatcher
-    def initialize(topic_entry, kafka, interval, parser, add_prefix, add_suffix, offset_manager, router, options={})
+    def initialize(topic_entry, kafka, interval, parser, add_prefix, add_suffix, offset_manager, router, kafka_message_key, options={})
       @topic_entry = topic_entry
       @kafka = kafka
       @callback = method(:consume)
@@ -209,6 +222,7 @@ class Fluent::KafkaInput < Fluent::Input
       @options = options
       @offset_manager = offset_manager
       @router = router
+      @kafka_message_key = kafka_message_key
 
       @next_offset = @topic_entry.offset
       if @topic_entry.offset == -1 && offset_manager
@@ -254,6 +268,9 @@ class Fluent::KafkaInput < Fluent::Input
           else
             record_time = Fluent::Engine.now
           end
+          if @kafka_message_key
+            record[@kafka_message_key] = msg.key
+          end
           es.add(record_time, record)
         rescue => e
           $log.warn "parser error in #{@topic_entry.topic}/#{@topic_entry.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset

data/lib/fluent/plugin/in_kafka_group.rb

@@ -25,6 +25,8 @@ class Fluent::KafkaGroupInput < Fluent::Input
                :desc => "Replace message timestamp with contents of 'time' field."
   config_param :time_format, :string, :default => nil,
                :desc => "Time format to be used to parse 'time' filed."
+  config_param :kafka_message_key, :string, :default => nil,
+               :desc => "Set kafka's message key to this field"
 
   config_param :retry_wait_seconds, :integer, :default => 30
   # Kafka consumer options
@@ -74,6 +76,10 @@ class Fluent::KafkaGroupInput < Fluent::Input
     config_array
   end
 
+  def multi_workers_ready?
+    true
+  end
+
   private :_config_to_array
 
   def configure(conf)
@@ -108,8 +114,14 @@ class Fluent::KafkaGroupInput < Fluent::Input
   def setup_parser
     case @format
     when 'json'
-      require 'yajl'
-      Proc.new { |msg| Yajl::Parser.parse(msg.value) }
+      begin
+        require 'oj'
+        Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
+        Proc.new { |msg| Oj.load(msg.value) }
+      rescue LoadError
+        require 'yajl'
+        Proc.new { |msg| Yajl::Parser.parse(msg.value) }
+      end
     when 'ltsv'
       require 'ltsv'
       Proc.new { |msg| LTSV.parse(msg.value, {:symbolize_keys => false}).first }
@@ -123,7 +135,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
 
   def start
     super
-   
+
     @kafka = Kafka.new(seed_brokers: @brokers,
                        ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                        ssl_client_cert: read_ssl_file(@ssl_client_cert),
@@ -189,6 +201,9 @@ class Fluent::KafkaGroupInput < Fluent::Input
               else
                 record_time = Fluent::Engine.now
               end
+              if @kafka_message_key
+                record[@kafka_message_key] = msg.key
+              end
               es.add(record_time, record)
             rescue => e
               log.warn "parser error in #{batch.topic}/#{batch.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset

data/lib/fluent/plugin/out_kafka.rb

@@ -149,6 +149,10 @@ DESC
     end
   end
 
+  def multi_workers_ready?
+    true
+  end
+
   def start
     super
     refresh_client

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -98,6 +98,10 @@ DESC
     @producers_mutex = Mutex.new
   end
 
+  def multi_workers_ready?
+    true
+  end
+
   def refresh_client(raise_error = true)
     if @zookeeper
       @seed_brokers = []
@@ -218,8 +222,14 @@ DESC
 
   def setup_formatter(conf)
     if @output_data_type == 'json'
-      require 'yajl'
-      Proc.new { |tag, time, record| Yajl::Encoder.encode(record) }
+      begin
+        require 'oj'
+        Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
+        Proc.new { |tag, time, record| Oj.dump(record) }
+      rescue LoadError
+        require 'yajl'
+        Proc.new { |tag, time, record| Yajl::Encoder.encode(record) }
+      end
     elsif @output_data_type == 'ltsv'
       require 'ltsv'
       Proc.new { |tag, time, record| LTSV.dump(record) }

data/test/plugin/test_out_kafka.rb

@@ -43,6 +43,12 @@ class KafkaOutputTest < Test::Unit::TestCase
     d = create_driver
   end
 
+  def test_mutli_worker_support
+    d = create_driver
+    assert_equal true, d.instance.multi_workers_ready?
+
+  end
+
   def test_write
     d = create_driver
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.6.3
+  version: 0.6.4
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-15 00:00:00.000000000 Z
+date: 2017-11-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -49,16 +49,22 @@ dependencies:
   name: ruby-kafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.4.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.4.1
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 1.0.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -113,7 +119,8 @@ files:
 - test/helper.rb
 - test/plugin/test_out_kafka.rb
 homepage: https://github.com/fluent/fluent-plugin-kafka
-licenses: []
+licenses:
+- Apache-2.0
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -131,7 +138,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.7.2
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin for Apache Kafka > 0.8