fluent-plugin-flume 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7572fd137da7ba6c6668e09fdb489ca58f417781
-  data.tar.gz: ec492fdb5fadfaddf89a21ea4b78f76c126bcd05
+  metadata.gz: 4b78497ad732b1daed34f85264299c5f531b8862
+  data.tar.gz: 01d60de145ffe1f44c682aa74ca6448e596d8a9e
 SHA512:
-  metadata.gz: b53cfe600d060d4537c2cac0b96b5cc4b020e9ad05b81b56e2bede3923836fb52612d7022cd6f90c64c211665c3499a07b1db8b97131188d19ef84b1c0fab1be
-  data.tar.gz: 7287e3c0988a562da41eac18cbbd535265d738af90b61015c639601b43c8bddcee5c9d2d476f7c15239242431f8b604869d208bdb3b7b136683189cd2940bf2e
+  metadata.gz: 34fe016d453e36b285731ac4882ad4d2001702f56a65921017bef3c383bdf1894526520e7c869136d64d67ec9c43dcb5c0575e6630c8e216de3c754e23a441df
+  data.tar.gz: 779f483c0eeae828af4b693df7d54a525277961cfb6f421a27a92d673664ee0873cd382b18f0754d4733770c775edd766df435995a05e435cf1b6952dd8ac6a0

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 0.1.3 - 2016/10/25
+
+	* Add format parameter for formatter plugins support
+
 Release 0.1.2 - 2015/11/30
 
 	* Updated thrift to 0.9.0

data/README.rdoc

@@ -83,7 +83,7 @@ Use flume-ng-fluentd-sink[https://github.com/cosmo0920/flume-ng-fluentd-sink] to
 
 == Flume Output
 
-Please add the following configurations to fluent.conf.  This allows fluentd to output its logs into another Flume server.  Note that fluentd conveys semi-structured data while Flume conveys unstructured data.  Thus the plugin translates semi-structured data into JSON data and conveys it to Flume.
+Please add the following configurations to fluent.conf.  This allows fluentd to output its logs into another Flume server.  Note that fluentd conveys semi-structured data while Flume conveys unstructured data.  Thus the plugin translates semi-structured data into JSON data by default and conveys it to Flume.  The format can be adjusted via formatters.
 
   # Flume output
   <match *>
@@ -98,6 +98,8 @@ These options are supported.
 * port: port number (default: 35863)
 * timeout: thrift protocol timeout (default: 30)
 * remove_prefix: prefix string, removed from the tag (default: nil)
+* format: The format of the thrift body (default: json)
+* trim_nl: Trim new line from thrift body (default: true)
 
 == Contributors
 

data/VERSION

@@ -1 +1 @@
-0.1.2
+0.1.3

data/lib/fluent/plugin/in_flume.rb

@@ -15,6 +15,9 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
+require 'fluent/input'
+
 module Fluent
 
 class FlumeInput < Input

data/lib/fluent/plugin/out_flume.rb

@@ -15,6 +15,9 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
+require 'fluent/output'
+
 module Fluent
 
 class FlumeOutput < BufferedOutput
@@ -25,6 +28,9 @@ class FlumeOutput < BufferedOutput
   config_param :timeout,   :integer, :default => 30
   config_param :remove_prefix,    :string, :default => nil
   config_param :default_category, :string, :default => 'unknown'
+  desc "The format of the thrift body. (default: json)"
+  config_param :format, :string, default: 'json'
+  config_param :trim_nl, :bool, default: true
 
   unless method_defined?(:log)
     define_method(:log) { $log }
@@ -42,7 +48,11 @@ class FlumeOutput < BufferedOutput
   def configure(conf)
     # override default buffer_chunk_limit
     conf['buffer_chunk_limit'] ||= '1m'
+
     super
+
+    @formatter = Plugin.new_formatter(@format)
+    @formatter.configure(conf)
   end
 
   def start
@@ -60,12 +70,12 @@ class FlumeOutput < BufferedOutput
 
   def format(tag, time, record)
     if @remove_prefix and
-        ( (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or
-          tag == @remove_prefix)
-      [(tag[@removed_length..-1] || @default_category), time, record].to_msgpack
-    else
-      [tag, time, record].to_msgpack
+        ((tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or tag == @remove_prefix)
+      tag = (tag[@removed_length..-1] || @default_category)
     end
+    fr = @formatter.format(tag, time, record)
+    fr.chomp! if @trim_nl
+    [tag, time, fr].to_msgpack
   end
 
   def write(chunk)
@@ -76,15 +86,15 @@ class FlumeOutput < BufferedOutput
     client = ThriftSourceProtocol::Client.new protocol
 
     count = 0
+    header = {}
     transport.open
     log.debug "thrift client opened: #{client}"
     begin
       chunk.msgpack_each { |tag, time, record|
-        entry = ThriftFlumeEvent.new(:body      => record.to_json.to_s.force_encoding('UTF-8'),
-                                     :headers   => {
-                                       'timestamp' => time.to_s,
-                                       'tag'       => tag,
-                                     })
+        header['timestamp'.freeze] = time.to_s
+        header['tag'.freeze] = tag
+        entry = ThriftFlumeEvent.new(:body    => record.force_encoding('UTF-8'),
+                                     :headers => header)
         client.append entry
         count += 1
       }

data/test/plugin/test_out_flume.rb

@@ -13,7 +13,7 @@ class FlumeOutputTest < Test::Unit::TestCase
   ]
 
   def create_driver(conf=CONFIG, tag='test')
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::FlumeOutput, tag)do
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::FlumeOutput, tag) do
       def write(chunk)
         chunk.read
       end
@@ -41,8 +41,9 @@ class FlumeOutputTest < Test::Unit::TestCase
     d = create_driver
     d.emit({"k11"=>"v11", "k12"=>"v12"}, time)
     d.emit({"k21"=>"v21", "k22"=>"v22"}, time)
-    d.expect_format [d.tag, time, {"k11"=>"v11", "k12"=>"v12"}].to_msgpack
-    d.expect_format [d.tag, time, {"k21"=>"v21", "k22"=>"v22"}].to_msgpack
+    puts ({"k11" => "v11", "k12" => "v12"}.to_json)
+    d.expect_format [d.tag, time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack
+    d.expect_format [d.tag, time, {"k21"=>"v21", "k22"=>"v22"}.to_json].to_msgpack
     d.run
 
     d = create_driver(CONFIG + %[
@@ -52,8 +53,8 @@ class FlumeOutputTest < Test::Unit::TestCase
 
     d.emit({"k11"=>"v11", "k12"=>"v12"}, time)
     d.emit({"k21"=>"v21", "k22"=>"v22"}, time)
-    d.expect_format ['flumeplugin', time, {"k11"=>"v11", "k12"=>"v12"}].to_msgpack
-    d.expect_format ['flumeplugin', time, {"k21"=>"v21", "k22"=>"v22"}].to_msgpack
+    d.expect_format ['flumeplugin', time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack
+    d.expect_format ['flumeplugin', time, {"k21"=>"v21", "k22"=>"v22"}.to_json].to_msgpack
     d.run
 
     d = create_driver(CONFIG + %[
@@ -61,7 +62,7 @@ class FlumeOutputTest < Test::Unit::TestCase
     ], 'xxx.test.flumeplugin')
     assert_equal 'xxx.test.flumeplugin', d.tag
     d.emit({"k11"=>"v11", "k12"=>"v12"}, time)
-    d.expect_format ['xxx.test.flumeplugin', time, {"k11"=>"v11", "k12"=>"v12"}].to_msgpack
+    d.expect_format ['xxx.test.flumeplugin', time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack
     d.run
 
     d = create_driver(CONFIG + %[
@@ -69,7 +70,7 @@ class FlumeOutputTest < Test::Unit::TestCase
     ], 'test')
     assert_equal 'test', d.tag
     d.emit({"k11"=>"v11", "k12"=>"v12"}, time)
-    d.expect_format ['unknown', time, {"k11"=>"v11", "k12"=>"v12"}].to_msgpack
+    d.expect_format ['unknown', time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack
     d.run
 
     d = create_driver(CONFIG + %[
@@ -78,8 +79,8 @@ class FlumeOutputTest < Test::Unit::TestCase
     assert_equal 'test.flumeplugin', d.tag
     d.emit({"k11"=>"v11", "k12"=>"v12"}, time)
     d.emit({"k21"=>"v21", "k22"=>"v22"}, time)
-    d.expect_format ['flumeplugin', time, {"k11"=>"v11", "k12"=>"v12"}].to_msgpack
-    d.expect_format ['flumeplugin', time, {"k21"=>"v21", "k22"=>"v22"}].to_msgpack
+    d.expect_format ['flumeplugin', time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack
+    d.expect_format ['flumeplugin', time, {"k21"=>"v21", "k22"=>"v22"}.to_json].to_msgpack
     d.run
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-flume
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Muga Nishizawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-01 00:00:00.000000000 Z
+date: 2016-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -116,7 +116,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Flume Input/Output plugin for Fluentd event collector