fluent-plugin-kafka 0.0.5 → 0.0.6

data/README.md

@@ -26,7 +26,7 @@ Or install it yourself as:
       host   <broker host>
       port   <broker port: default=9092>
       topics <listening topics(separate with comma',')>
-      format <input text type (text|json)>
+      format <input text type (text|json|ltsv)>
       add_prefix <tag prefix (Optional)>
       add_suffix <tag suffix (Optional)>
     </source>
@@ -38,6 +38,8 @@ Or install it yourself as:
       brokers          <broker1_host>:<broker1_ip>,<broker2_host>:<broker2_ip>,..
       default_topic    <output topic>
       output_data_type (json|ltsv|attr:<record name>)
+      output_include_tag  (true|false) :default => false
+      output_include_time (true|false) :default => false
     </match>
 
 ### Buffered output plugin
@@ -49,6 +51,8 @@ Or install it yourself as:
       flush_interval   <flush interval (sec) :default => 60>
       buffer_type      (file|memory)
       output_data_type (json|ltsv|attr:<record name>)
+      output_include_tag  (true|false) :default => false
+      output_include_time (true|false) :default => false
     </match>
 
 ## Contributing

data/fluent-plugin-kafka.gemspec

@@ -12,9 +12,9 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version = '0.0.5'
+  gem.version = '0.0.6'
   gem.add_dependency 'fluentd'
   gem.add_dependency 'poseidon'
   gem.add_dependency 'ltsv'
-  gem.add_dependency 'json'
+  gem.add_dependency 'yajl-ruby'
 end

data/lib/fluent/plugin/in_kafka.rb

@@ -3,7 +3,7 @@ module Fluent
 class KafkaInput < Input
   Plugin.register_input('kafka', self)
 
-  config_param :format, :string, :default => 'json' # (json|text)
+  config_param :format, :string, :default => 'json' # (json|text|ltsv)
   config_param :host, :string, :default => 'localhost'
   config_param :port, :integer, :default => 2181
   config_param :interval, :integer, :default => 1 # seconds
@@ -28,7 +28,9 @@ class KafkaInput < Input
 
     case @format
     when 'json'
-      require 'json'
+      require 'yajl'
+    when 'ltsv'
+      require 'ltsv'
     end
   end
 
@@ -105,7 +107,9 @@ class KafkaInput < Input
       parsed_record = {}
       case @format
       when 'json'
-        parsed_record = JSON.parse(record)
+        parsed_record = Yajl::Parser.parse(record)
+      when 'ltsv'
+        parsed_record = LTSV.parse(record)
       when 'text'
         parsed_record = record
       end

data/lib/fluent/plugin/out_kafka.rb

@@ -11,6 +11,8 @@ class Fluent::KafkaOutput < Fluent::Output
   config_param :default_partition, :integer, :default => 0
   config_param :client_id, :string, :default => 'kafka'
   config_param :output_data_type, :string, :default => 'json'
+  config_param :output_include_tag, :bool, :default => false
+  config_param :output_include_time, :bool, :default => false
   attr_accessor :output_data_type
   attr_accessor :field_separator
 
@@ -20,7 +22,7 @@ class Fluent::KafkaOutput < Fluent::Output
     @producers = {} # keyed by topic:partition
     case @output_data_type
     when 'json'
-      require 'json'
+      require 'yajl'
     when 'ltsv'
       require 'ltsv'
     end
@@ -56,13 +58,15 @@ class Fluent::KafkaOutput < Fluent::Output
     if @custom_attributes.nil?
       case @output_data_type
       when 'json'
-        JSON.dump(record)
+        Yajl::Encoder.encode(record)
       when 'ltsv'
         LTSV.dump(record)
       else
         record.to_s
       end
     else
+      @custom_attributes.unshift('time') if @output_include_time
+      @custom_attributes.unshift('tag') if @output_include_tag
       @custom_attributes.map { |attr|
         record[attr].nil? ? '' : record[attr].to_s
       }.join(@f_separator)
@@ -72,6 +76,8 @@ class Fluent::KafkaOutput < Fluent::Output
   def emit(tag, es, chain)
     chain.next
     es.each do |time,record|
+      record['time'] = time if @output_include_time
+      record['tag'] = tag if @output_include_tag
       topic = record['topic'] || self.default_topic || tag
       partition = record['partition'] || self.default_partition
       message = Poseidon::MessageToSend.new(topic, parse_record(record))

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -1,3 +1,4 @@
+# encode: utf-8
 class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
   Fluent::Plugin.register_output('kafka_buffered', self)
 
@@ -11,6 +12,8 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
   config_param :default_partition, :integer, :default => 0
   config_param :client_id, :string, :default => 'kafka'
   config_param :output_data_type, :string, :default => 'json'
+  config_param :output_include_tag, :bool, :default => false
+  config_param :output_include_time, :bool, :default => false
   attr_accessor :output_data_type
   attr_accessor :field_separator
 
@@ -20,7 +23,7 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
     @producers = {} # keyed by topic:partition
     case @output_data_type
     when 'json'
-      require 'json'
+      require 'yajl'
     when 'ltsv'
       require 'ltsv'
     end
@@ -59,13 +62,15 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
     if @custom_attributes.nil?
       case @output_data_type
       when 'json'
-        JSON.dump(record)
+        Yajl::Encoder.encode(record)
       when 'ltsv'
         LTSV.dump(record)
       else
         record.to_s
       end
     else
+      @custom_attributes.unshift('time') if @output_include_time
+      @custom_attributes.unshift('tag') if @output_include_tag
       @custom_attributes.map { |attr|
         record[attr].nil? ? '' : record[attr].to_s
       }.join(@f_separator)
@@ -75,6 +80,8 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
   def write(chunk)
     records_by_topic = {}
     chunk.msgpack_each { |tag, time, record|
+      record['time'] = time if @output_include_time
+      record['tag'] = tag if @output_include_tag
       topic = record['topic'] || self.default_topic || tag
       partition = record['partition'] || self.default_partition
       message = Poseidon::MessageToSend.new(topic, parse_record(record))

metadata

@@ -1,69 +1,78 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
+  prerelease: 
 platform: ruby
 authors:
 - Hidemasa Togashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-05 00:00:00.000000000 Z
+date: 2014-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: poseidon
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: ltsv
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: json
+  name: yajl-ruby
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 description: Fluentd plugin for Apache Kafka > 0.8
@@ -85,26 +94,27 @@ files:
 - test/plugin/test_out_kafka.rb
 homepage: https://github.com/htgc/fluent-plugin-kafka
 licenses: []
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 1.8.23
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: Fluentd plugin for Apache Kafka > 0.8
 test_files:
 - test/helper.rb

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: e7ea6df9ab262c3d1a76fd1bf8084dcacfed5e13
-  data.tar.gz: a4e75c5d1c30425d35a7cc57a802256dd28901fb
-SHA512:
-  metadata.gz: 12b5a1579ffd62f45279983cbbbc3c13c4659a0c736b29b6f8e40c765b839ed3a0e6424be27e12917b674f6f62bff076c5879de3cfe133dc9b8b269f8aebc489
-  data.tar.gz: 963e4236af8e79c47eecfdfd0b58f73c6a9e68f3161b1c984d48691ae025c64d3d5e57bb18f8a20ce91430d11ad4007ed6893c7e20aa31efded4ad7d9dd24192