fluent-plugin-flume 0.2.0 → 1.0.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d92aa659b736a1e4228653bcf7089cf0844ed412
-  data.tar.gz: 74f103fa74799fdb9ee89debad3a3ce803c360ae
+  metadata.gz: 5c6c30fffafba5fabb5604d70ec69217d014e15b
+  data.tar.gz: 39e66e209a62569b43d5a5c9f505dab574918db0
 SHA512:
-  metadata.gz: d1ec6c1fdd6ab7cd36acc3bdef637f4186c4464adec65839abeea0116a1e464240dccf2f88c414127cf88ec6dbea340347eac0fc7f6fbd9dee87012820b50ef6
-  data.tar.gz: fe24084964f0b71d51073c807ede72419dfde1faca46f824b5dbbc7b73875e6c67fba1392905f55cf0254749ae8ae0d1a08d898ce7eb998fb0670f30fe9601b7
+  metadata.gz: a1fbfb352b40f8f0d8822a085fc415fc376c834c2e44e2962840140c8cc4cb0b82ad72ccb8281e58a904fa119b3ef2dfdb9508536135fa49b4146dd6f04e0a3c
+  data.tar.gz: 59b805d33056fd27735e6c9d75c118c5453e854e40ee3a9dc709c4dd9185f3346e74a3ad2d222ae8e7debf28c60e876536fc4870a392e62c8c1d7f50095b5f8a

data/VERSION

@@ -1 +1 @@
-0.2.0
+1.0.0.rc1

data/lib/fluent/plugin/out_flume.rb

@@ -16,21 +16,35 @@
 #    limitations under the License.
 #
 
-require 'fluent/output'
+require 'fluent/plugin/output'
 
-module Fluent
-  class FlumeOutput < BufferedOutput
+module Fluent::Plugin
+  class FlumeOutput < Output
     Fluent::Plugin.register_output('flume', self)
 
+    helpers :formatter, :compat_parameters
+
+    DEFAULT_BUFFER_TYPE = "memory"
+    DEFAULT_FORMAT_TYPE = 'json'
+
     config_param :host,      :string,  :default => 'localhost'
     config_param :port,      :integer, :default => 35863
     config_param :timeout,   :integer, :default => 30
     config_param :remove_prefix,    :string, :default => nil
     config_param :default_category, :string, :default => 'unknown'
     desc "The format of the thrift body. (default: json)"
-    config_param :format, :string, default: 'json'
+    config_param :format, :string, default: DEFAULT_FORMAT_TYPE
     config_param :trim_nl, :bool, default: true
 
+    config_section :buffer do
+      config_set_default :@type, DEFAULT_BUFFER_TYPE
+      config_set_default :chunk_keys, ['tag']
+    end
+
+    config_section :format do
+      config_set_default :@type, DEFAULT_FORMAT_TYPE
+    end
+
     unless method_defined?(:log)
       define_method(:log) { $log }
     end
@@ -47,11 +61,10 @@ module Fluent
     def configure(conf)
       # override default buffer_chunk_limit
       conf['buffer_chunk_limit'] ||= '1m'
-
+      compat_parameters_convert(conf, :formatter)
       super
 
-      @formatter = Plugin.new_formatter(@format)
-      @formatter.configure(conf)
+      @formatter = formatter_create
     end
 
     def start
@@ -77,6 +90,14 @@ module Fluent
       [time, fr].to_msgpack
     end
 
+    def formatted_to_msgpack_binary
+      true
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
     def write(chunk)
       socket = Thrift::Socket.new @host, @port, @timeout
       transport = Thrift::FramedTransport.new socket
@@ -84,7 +105,7 @@ module Fluent
       protocol = Thrift::CompactProtocol.new transport
       client = ThriftSourceProtocol::Client.new protocol
 
-      tag = chunk.key
+      tag = chunk.metadata.tag
       count = 0
       header = {}
       transport.open

data/test/plugin/test_out_flume.rb

@@ -1,8 +1,12 @@
 require 'test/unit'
 require 'fluent/test'
+require 'fluent/test/helpers'
+require 'fluent/test/driver/output'
 require 'lib/fluent/plugin/out_flume'
 
 class FlumeOutputTest < Test::Unit::TestCase
+  include Fluent::Test::Helpers
+
   def setup
     Fluent::Test.setup
   end
@@ -12,8 +16,8 @@ class FlumeOutputTest < Test::Unit::TestCase
     port 35862
   ]
 
-  def create_driver(conf=CONFIG, tag='test')
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::FlumeOutput, tag) do
+  def create_driver(conf=CONFIG)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::FlumeOutput) do
       def write(chunk)
         chunk.read
       end
@@ -36,79 +40,82 @@ class FlumeOutputTest < Test::Unit::TestCase
   end
 
   def test_format
-    time = Time.parse("2011-12-21 13:14:15 UTC").to_i
+    time = event_time("2011-12-21 13:14:15 UTC")
 
     d = create_driver
-    d.emit({"k11"=>"v11", "k12"=>"v12"}, time)
-    d.emit({"k21"=>"v21", "k22"=>"v22"}, time)
-    d.expect_format [time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack
-    d.expect_format [time, {"k21"=>"v21", "k22"=>"v22"}.to_json].to_msgpack
-    d.run
+    d.run(default_tag: 'test') do
+      d.feed(time, {"k11"=>"v11", "k12"=>"v12"})
+      d.feed(time, {"k21"=>"v21", "k22"=>"v22"})
+    end
+    assert_equal [time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack, d.formatted[0]
+    assert_equal [time, {"k21"=>"v21", "k22"=>"v22"}.to_json].to_msgpack, d.formatted[1]
 
     d = create_driver(CONFIG + %[
       remove_prefix test
-    ], 'test.flumeplugin')
-    assert_equal 'test.flumeplugin', d.tag
+    ])
 
-    d.emit({"k11"=>"v11", "k12"=>"v12"}, time)
-    d.emit({"k21"=>"v21", "k22"=>"v22"}, time)
-    d.expect_format [time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack
-    d.expect_format [time, {"k21"=>"v21", "k22"=>"v22"}.to_json].to_msgpack
-    d.run
+    d.run(default_tag: 'test.flumeplugin') do
+      d.feed(time, {"k11"=>"v11", "k12"=>"v12"})
+      d.feed(time, {"k21"=>"v21", "k22"=>"v22"})
+    end
+    assert_equal [time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack, d.formatted[0]
+    assert_equal [time, {"k21"=>"v21", "k22"=>"v22"}.to_json].to_msgpack, d.formatted[1]
 
     d = create_driver(CONFIG + %[
       remove_prefix test
-    ], 'xxx.test.flumeplugin')
-    assert_equal 'xxx.test.flumeplugin', d.tag
-    d.emit({"k11"=>"v11", "k12"=>"v12"}, time)
-    d.expect_format [time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack
-    d.run
+    ])
+    d.run(default_tag: 'xxx.test.flumeplugin') do
+      d.feed(time, {"k11"=>"v11", "k12"=>"v12"})
+    end
 
     d = create_driver(CONFIG + %[
       remove_prefix test
-    ], 'test')
-    assert_equal 'test', d.tag
-    d.emit({"k11"=>"v11", "k12"=>"v12"}, time)
-    d.expect_format [time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack
-    d.run
+    ])
+    d.run(default_tag: 'test') do
+      d.feed(time, {"k11"=>"v11", "k12"=>"v12"})
+    end
+    assert_equal [time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack, d.formatted[0]
 
     d = create_driver(CONFIG + %[
       remove_prefix test
-    ], 'test.flumeplugin')
-    assert_equal 'test.flumeplugin', d.tag
-    d.emit({"k11"=>"v11", "k12"=>"v12"}, time)
-    d.emit({"k21"=>"v21", "k22"=>"v22"}, time)
-    d.expect_format [time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack
-    d.expect_format [time, {"k21"=>"v21", "k22"=>"v22"}.to_json].to_msgpack
-    d.run
+    ])
+    d.run(default_tag: 'test.flumeplugin') do
+      d.feed(time, {"k11"=>"v11", "k12"=>"v12"})
+      d.feed(time, {"k21"=>"v21", "k22"=>"v22"})
+    end
+    assert_equal [time, {"k11"=>"v11", "k12"=>"v12"}.to_json].to_msgpack, d.formatted[0]
+    assert_equal [time, {"k21"=>"v21", "k22"=>"v22"}.to_json].to_msgpack, d.formatted[1]
   end
 
   def test_write
-    time = Time.parse("2011-12-21 13:14:15 UTC").to_i
+    time = event_time("2011-12-21 13:14:15 UTC")
 
     d = create_driver
-    d.emit({"k11"=>"v11", "k12"=>"v12"}, time)
-    d.emit({"k21"=>"v21", "k22"=>"v22"}, time)
-    d.run
+    d.run(default_tag: 'test') do
+      d.feed(time, {"k11"=>"v11", "k12"=>"v12"})
+      d.feed(time, {"k21"=>"v21", "k22"=>"v22"})
+    end
 
     d = create_driver(CONFIG + %[
       remove_prefix test
-    ], 'test.flumeplugin')
-    assert_equal 'test.flumeplugin', d.tag
-    d.run
+    ])
+    d.end_if do
+      d.events.size >= 0
+    end
+    d.run(default_tag: 'test.flumeplugin')
 
     d = create_driver(CONFIG + %[
       remove_prefix test
-    ], 'xxx.test.flumeplugin')
-    assert_equal 'xxx.test.flumeplugin', d.tag
-    d.emit({"k11"=>"v11", "k12"=>"v12"}, time)
-    d.run
+    ])
+    d.run(default_tag: 'xxx.test.flumeplugin') do
+      d.feed(time, {"k11"=>"v11", "k12"=>"v12"})
+    end
 
     d = create_driver(CONFIG + %[
       remove_prefix test
-    ], 'test')
-    assert_equal 'test', d.tag
-    d.emit({"k11"=>"v11", "k12"=>"v12"}, time)
-    d.run
+    ])
+    d.run(default_tag: 'test') do
+      d.feed(time, {"k11"=>"v11", "k12"=>"v12"})
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-flume
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 1.0.0.rc1
 platform: ruby
 authors:
 - Muga Nishizawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-25 00:00:00.000000000 Z
+date: 2017-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -115,12 +115,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Flume Input/Output plugin for Fluentd event collector