fluentd 0.12.40 → 0.14.0

data/test/plugin/test_in_tcp.rb

@@ -1,7 +1,20 @@
 require_relative '../helper'
 require 'fluent/test'
+require 'fluent/plugin/in_tcp'
 
 class TcpInputTest < Test::Unit::TestCase
+  class << self
+    def startup
+      socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+      @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
+      ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+    end
+
+    def shutdown
+      @server.close
+    end
+  end
+
   def setup
     Fluent::Test.setup
   end
@@ -83,6 +96,7 @@ class TcpInputTest < Test::Unit::TestCase
     assert_equal(2, emits.size)
     emits.each_index {|i|
       assert_equal(tests[i]['expected'], emits[i][2]['message'])
+      assert(emits[i][1].is_a?(Fluent::EventTime))
     }
   end
 end

data/test/plugin/test_in_udp.rb

@@ -1,7 +1,20 @@
 require_relative '../helper'
 require 'fluent/test'
+require 'fluent/plugin/in_udp'
 
 class UdpInputTest < Test::Unit::TestCase
+  class << self
+    def startup
+      socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+      @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
+      ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+    end
+
+    def shutdown
+      @server.close
+    end
+  end
+
   def setup
     Fluent::Test.setup
   end
@@ -32,19 +45,10 @@ class UdpInputTest < Test::Unit::TestCase
       d = create_driver(v)
       assert_equal PORT, d.instance.port
       assert_equal k, d.instance.bind
-      assert_equal 4096, d.instance.message_length_limit
+      assert_equal 4096, d.instance.body_size_limit
     }
   end
 
-  data(
-    'message_length_limit' => 'message_length_limit 2048',
-    'body_size_limit' => 'body_size_limit 2048'
-  )
-  test 'message_length_limit/body_size_limit compatibility' do |param|
-    d = create_driver(CONFIG + param)
-    assert_equal 2048, d.instance.message_length_limit
-  end
-
   def test_time_format
     configs = {'127.0.0.1' => CONFIG}
     configs.merge!('::1' => IPv6_CONFIG) if ipv6_enabled?
@@ -53,8 +57,8 @@ class UdpInputTest < Test::Unit::TestCase
       d = create_driver(v)
 
       tests = [
-        {'msg' => '[Sep 11 00:00:00] localhost logger: foo', 'expected' => Time.strptime('Sep 11 00:00:00', '%b %d %H:%M:%S').to_i},
-        {'msg' => '[Sep  1 00:00:00] localhost logger: foo', 'expected' => Time.strptime('Sep  1 00:00:00', '%b  %d %H:%M:%S').to_i},
+        {'msg' => '[Sep 11 00:00:00] localhost logger: foo', 'expected' => Fluent::EventTime.from_time(Time.strptime('Sep 11 00:00:00', '%b %d %H:%M:%S'))},
+        {'msg' => '[Sep  1 00:00:00] localhost logger: foo', 'expected' => Fluent::EventTime.from_time(Time.strptime('Sep  1 00:00:00', '%b  %d %H:%M:%S'))},
       ]
 
       d.run do
@@ -63,14 +67,16 @@ class UdpInputTest < Test::Unit::TestCase
         tests.each {|test|
           u.send(test['msg'], 0)
         }
+        u.close
         sleep 1
       end
 
       emits = d.emits
       emits.each_index {|i|
-        assert_equal(tests[i]['expected'], emits[i][1])
+        assert_equal_event_time(tests[i]['expected'], emits[i][1])
       }
     }
+
   end
 
   {
@@ -97,6 +103,7 @@ class UdpInputTest < Test::Unit::TestCase
         tests.each { |test|
           u.send(test['msg'], 0)
         }
+        u.close
         sleep 1
       end
 
@@ -108,6 +115,7 @@ class UdpInputTest < Test::Unit::TestCase
     assert_equal(2, emits.size)
     emits.each_index {|i|
       assert_equal(tests[i]['expected'], emits[i][2]['message'])
+      assert(emits[i][1].is_a?(Fluent::EventTime))
     }
   end
 end

data/test/plugin/test_input.rb

@@ -0,0 +1,122 @@
+require_relative '../helper'
+require 'fluent/plugin/input'
+require 'flexmock/test_unit'
+
+module FluentPluginInputTest
+  class DummyPlugin < Fluent::Plugin::Input
+  end
+end
+
+class InputTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+    @p = FluentPluginInputTest::DummyPlugin.new
+  end
+
+  test 'has healthy lifecycle' do
+    assert !@p.configured?
+    @p.configure(config_element())
+    assert @p.configured?
+
+    assert !@p.started?
+    @p.start
+    assert @p.start
+
+    assert !@p.stopped?
+    @p.stop
+    assert @p.stopped?
+
+    assert !@p.before_shutdown?
+    @p.before_shutdown
+    assert @p.before_shutdown?
+
+    assert !@p.shutdown?
+    @p.shutdown
+    assert @p.shutdown?
+
+    assert !@p.after_shutdown?
+    @p.after_shutdown
+    assert @p.after_shutdown?
+
+    assert !@p.closed?
+    @p.close
+    assert @p.closed?
+
+    assert !@p.terminated?
+    @p.terminate
+    assert @p.terminated?
+  end
+
+  test 'has plugin_id automatically generated' do
+    assert @p.respond_to?(:plugin_id_configured?)
+    assert @p.respond_to?(:plugin_id)
+
+    @p.configure(config_element())
+
+    assert !@p.plugin_id_configured?
+    assert @p.plugin_id
+    assert{ @p.plugin_id != 'mytest' }
+  end
+
+  test 'has plugin_id manually configured' do
+    @p.configure(config_element('ROOT', '', {'@id' => 'mytest'}))
+    assert @p.plugin_id_configured?
+    assert_equal 'mytest', @p.plugin_id
+  end
+
+  test 'has plugin logger' do
+    assert @p.respond_to?(:log)
+    assert @p.log
+
+    # default logger
+    original_logger = @p.log
+
+    @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
+
+    assert{ @p.log.object_id != original_logger.object_id }
+    assert_equal Fluent::Log::LEVEL_DEBUG, @p.log.level
+  end
+
+  test 'can load plugin helpers' do
+    assert_nothing_raised do
+      class FluentPluginInputTest::DummyPlugin2 < Fluent::Plugin::Input
+        helpers :storage
+      end
+    end
+  end
+
+  test 'has router and can emit into it' do
+    assert @p.has_router?
+
+    @p.configure(config_element())
+    assert @p.router
+
+    DummyRouter = Struct.new(:emits) do
+      def emit(tag, es)
+        self.emits << [tag, es]
+      end
+    end
+    @p.router = DummyRouter.new([])
+    @p.router.emit('mytag', [])
+    @p.router.emit('mytag.testing', ['it is not es, but no problem for tests'])
+
+    assert_equal ['mytag', []], @p.router.emits[0]
+    assert_equal ['mytag.testing', ['it is not es, but no problem for tests']], @p.router.emits[1]
+  end
+
+  test 'has router for specified label if configured' do
+    @p.configure(config_element())
+    original_router = @p.router
+
+    router_mock = flexmock('mytest')
+    router_mock.should_receive(:emit).once.with('mytag.testing', ['for mock'])
+    label_mock = flexmock('mylabel')
+    label_mock.should_receive(:event_router).once.and_return(router_mock)
+    Fluent::Engine.root_agent.labels['@mytest'] = label_mock
+
+    @p.configure(config_element('ROOT', '', {'@label' => '@mytest'}))
+    assert{ @p.router.object_id != original_router.object_id }
+
+    @p.router.emit('mytag.testing', ['for mock'])
+  end
+end

data/test/plugin/test_multi_output.rb

@@ -0,0 +1,180 @@
+require_relative '../helper'
+require 'fluent/plugin/multi_output'
+require 'fluent/event'
+
+require 'json'
+require 'time'
+require 'timeout'
+
+module FluentPluginMultiOutputTest
+  class DummyMultiOutput < Fluent::Plugin::MultiOutput
+    attr_reader :events
+    def initialize
+      super
+      @events = []
+    end
+    def configure(conf)
+      super
+    end
+    def process(tag, es)
+      es.each do |time, record|
+        @events << [tag, time, record]
+      end
+    end
+  end
+  class DummyCompatMultiOutput < Fluent::Plugin::MultiOutput
+    def initialize
+      super
+      @compat = true
+    end
+    def configure(conf)
+      super
+    end
+    def process(tag, es)
+      # ...
+    end
+  end
+
+  class Dummy1Output < Fluent::Plugin::Output
+    Fluent::Plugin.register_output('dummy_test_multi_output_1', self)
+    attr_reader :configured
+    def configure(conf)
+      super
+      @configured = true
+    end
+    def process(tag, es)
+    end
+  end
+  class Dummy2Output < Fluent::Plugin::Output
+    Fluent::Plugin.register_output('dummy_test_multi_output_2', self)
+    attr_reader :configured
+    def configure(conf)
+      super
+      @configured = true
+    end
+    def process(tag, es)
+    end
+  end
+  class Dummy3Output < Fluent::Plugin::Output
+    Fluent::Plugin.register_output('dummy_test_multi_output_3', self)
+    attr_reader :configured
+    def configure(conf)
+      super
+      @configured = true
+    end
+    def process(tag, es)
+    end
+  end
+  class Dummy4Output < Fluent::Plugin::Output
+    Fluent::Plugin.register_output('dummy_test_multi_output_4', self)
+    attr_reader :configured
+    def configure(conf)
+      super
+      @configured = true
+    end
+    def process(tag, es)
+    end
+  end
+end
+
+class MultiOutputTest < Test::Unit::TestCase
+  def create_output(type=:multi)
+    case type
+    when :compat_multi
+      FluentPluginMultiOutputTest::DummyCompatMultiOutput.new
+    else
+      FluentPluginMultiOutputTest::DummyMultiOutput.new
+    end
+  end
+
+  sub_test_case 'basic multi output plugin' do
+    setup do
+      Fluent::Test.setup
+      @i = create_output()
+    end
+
+    teardown do
+      @i.log.out.reset
+    end
+
+    test '#configure raises error if <store> sections are missing' do
+      conf = config_element('ROOT', '', { '@type' => 'dummy_test_multi_output' }, [])
+      assert_raise Fluent::ConfigError do
+        @i.configure(conf)
+      end
+    end
+
+    test '#configure initialize child plugins and call these #configure' do
+      assert_equal [], @i.outputs
+
+      conf = config_element('ROOT', '', { '@type' => 'dummy_test_multi_output' },
+        [
+          config_element('store', '', { '@type' => 'dummy_test_multi_output_1' }),
+          config_element('store', '', { '@type' => 'dummy_test_multi_output_2' }),
+          config_element('store', '', { '@type' => 'dummy_test_multi_output_3' }),
+          config_element('store', '', { '@type' => 'dummy_test_multi_output_4' }),
+        ]
+      )
+      @i.configure(conf)
+
+      assert_equal 4, @i.outputs.size
+
+      assert @i.outputs[0].is_a? FluentPluginMultiOutputTest::Dummy1Output
+      assert @i.outputs[0].configured
+
+      assert @i.outputs[1].is_a? FluentPluginMultiOutputTest::Dummy2Output
+      assert @i.outputs[1].configured
+
+      assert @i.outputs[2].is_a? FluentPluginMultiOutputTest::Dummy3Output
+      assert @i.outputs[2].configured
+
+      assert @i.outputs[3].is_a? FluentPluginMultiOutputTest::Dummy4Output
+      assert @i.outputs[3].configured
+    end
+
+    test '#configure warns if "type" is used in <store> sections instead of "@type"' do
+      assert_equal [], @i.log.out.logs
+
+      conf = config_element('ROOT', '', { '@type' => 'dummy_test_multi_output' },
+        [
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_1' }),
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_2' }),
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_3' }),
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_4' }),
+        ]
+      )
+      @i.configure(conf)
+      assert_equal 4, @i.outputs.size
+
+      logs = @i.log.out.logs
+      assert{ logs.select{|log| log.include?('[warn]') && log.include?("'type' is deprecated parameter name. use '@type' instead.") }.size == 4 }
+    end
+
+    test '#emit_events calls #process always' do
+      conf = config_element('ROOT', '', { '@type' => 'dummy_test_multi_output' },
+        [
+          config_element('store', '', { '@type' => 'dummy_test_multi_output_1' }),
+          config_element('store', '', { '@type' => 'dummy_test_multi_output_2' }),
+          config_element('store', '', { '@type' => 'dummy_test_multi_output_3' }),
+          config_element('store', '', { '@type' => 'dummy_test_multi_output_4' }),
+        ]
+      )
+      @i.configure(conf)
+      @i.start
+
+      assert @i.events.empty?
+
+      @i.emit_events(
+        'test.tag',
+        Fluent::ArrayEventStream.new(
+          [
+            [event_time(), {"message" => "multi test 1"}],
+            [event_time(), {"message" => "multi test 1"}],
+          ]
+        )
+      )
+
+      assert_equal 2, @i.events.size
+    end
+  end
+end

data/test/plugin/test_out_buffered_null.rb

@@ -0,0 +1,79 @@
+require_relative '../helper'
+require 'fluent/test/driver/output'
+require 'fluent/plugin/out_buffered_null'
+
+class BufferedNullOutputTestCase < Test::Unit::TestCase
+  sub_test_case 'BufferedNullOutput' do
+    test 'default chunk limit size is 100' do
+      d = Fluent::Test::Driver::Output.new(Fluent::Plugin::BufferedNullOutput).configure('')
+      assert_equal 10 * 1024, d.instance.buffer_config.chunk_limit_size
+      assert d.instance.buffer_config.flush_at_shutdown
+      assert_equal ['tag'], d.instance.buffer_config.chunk_keys
+      assert d.instance.chunk_key_tag
+      assert !d.instance.chunk_key_time
+      assert_equal [], d.instance.chunk_keys
+    end
+
+    test 'writes standard formattted chunks' do
+      d = Fluent::Test::Driver::Output.new(Fluent::Plugin::BufferedNullOutput).configure('')
+      t = event_time("2016-05-23 00:22:13 -0800")
+      d.run(default_tag: 'test', flush: true) do
+        d.feed(t, {"message" => "null null null"})
+        d.feed(t, {"message" => "null null"})
+        d.feed(t, {"message" => "null"})
+      end
+
+      assert_equal 3, d.instance.emit_count
+      assert_equal 3, d.instance.emit_records
+    end
+
+    test 'check for chunk passed to #write' do
+      d = Fluent::Test::Driver::Output.new(Fluent::Plugin::BufferedNullOutput).configure('')
+      data = []
+      d.instance.feed_proc = ->(chunk){ data << [chunk.unique_id, chunk.metadata.tag, chunk.read] }
+
+      t = event_time("2016-05-23 00:22:13 -0800")
+      d.run(default_tag: 'test', flush: true) do
+        d.feed(t, {"message" => "null null null"})
+        d.feed(t, {"message" => "null null"})
+        d.feed(t, {"message" => "null"})
+      end
+
+      assert_equal 1, data.size
+      _, tag, binary = data.first
+      events = []
+      Fluent::MessagePackFactory.unpacker.feed_each(binary){|obj| events << obj }
+      assert_equal 'test', tag
+      assert_equal [ [t, {"message" => "null null null"}], [t, {"message" => "null null"}], [t, {"message" => "null"}] ], events
+    end
+
+    test 'check for chunk passed to #try_write' do
+      d = Fluent::Test::Driver::Output.new(Fluent::Plugin::BufferedNullOutput).configure('')
+      data = []
+      d.instance.feed_proc = ->(chunk){ data << [chunk.unique_id, chunk.metadata.tag, chunk.read] }
+      d.instance.delayed = true
+
+      t = event_time("2016-05-23 00:22:13 -0800")
+      d.run(default_tag: 'test', flush: true, shutdown: false) do
+        d.feed(t, {"message" => "null null null"})
+        d.feed(t, {"message" => "null null"})
+        d.feed(t, {"message" => "null"})
+      end
+
+      assert_equal 1, data.size
+      chunk_id, tag, binary = data.first
+      events = []
+      Fluent::MessagePackFactory.unpacker.feed_each(binary){|obj| events << obj }
+      assert_equal 'test', tag
+      assert_equal [ [t, {"message" => "null null null"}], [t, {"message" => "null null"}], [t, {"message" => "null"}] ], events
+
+      assert_equal [chunk_id], d.instance.buffer.dequeued.keys
+
+      d.instance.commit_write(chunk_id)
+
+      assert_equal [], d.instance.buffer.dequeued.keys
+
+      d.instance_shutdown
+    end
+  end
+end