fluentd 0.12.43 → 0.14.0

data/test/plugin/test_in_tcp.rb

@@ -1,7 +1,20 @@
 require_relative '../helper'
 require 'fluent/test'
+require 'fluent/plugin/in_tcp'
 
 class TcpInputTest < Test::Unit::TestCase
+  class << self
+    def startup
+      socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+      @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
+      ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+    end
+
+    def shutdown
+      @server.close
+    end
+  end
+
   def setup
     Fluent::Test.setup
   end
@@ -83,6 +96,7 @@ class TcpInputTest < Test::Unit::TestCase
     assert_equal(2, emits.size)
     emits.each_index {|i|
       assert_equal(tests[i]['expected'], emits[i][2]['message'])
+      assert(emits[i][1].is_a?(Fluent::EventTime))
     }
   end
 end

data/test/plugin/test_in_udp.rb

@@ -1,7 +1,20 @@
 require_relative '../helper'
 require 'fluent/test'
+require 'fluent/plugin/in_udp'
 
 class UdpInputTest < Test::Unit::TestCase
+  class << self
+    def startup
+      socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+      @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
+      ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+    end
+
+    def shutdown
+      @server.close
+    end
+  end
+
   def setup
     Fluent::Test.setup
   end
@@ -32,20 +45,10 @@ class UdpInputTest < Test::Unit::TestCase
       d = create_driver(v)
       assert_equal PORT, d.instance.port
       assert_equal k, d.instance.bind
-      assert_equal 4096, d.instance.message_length_limit
-      assert_equal nil, d.instance.receive_buffer_size
+      assert_equal 4096, d.instance.body_size_limit
     }
   end
 
-  data(
-    'message_length_limit' => 'message_length_limit 2048',
-    'body_size_limit' => 'body_size_limit 2048'
-  )
-  test 'message_length_limit/body_size_limit compatibility' do |param|
-    d = create_driver(CONFIG + param)
-    assert_equal 2048, d.instance.message_length_limit
-  end
-
   def test_time_format
     configs = {'127.0.0.1' => CONFIG}
     configs.merge!('::1' => IPv6_CONFIG) if ipv6_enabled?
@@ -54,8 +57,8 @@ class UdpInputTest < Test::Unit::TestCase
       d = create_driver(v)
 
       tests = [
-        {'msg' => '[Sep 11 00:00:00] localhost logger: foo', 'expected' => Time.strptime('Sep 11 00:00:00', '%b %d %H:%M:%S').to_i},
-        {'msg' => '[Sep  1 00:00:00] localhost logger: foo', 'expected' => Time.strptime('Sep  1 00:00:00', '%b  %d %H:%M:%S').to_i},
+        {'msg' => '[Sep 11 00:00:00] localhost logger: foo', 'expected' => Fluent::EventTime.from_time(Time.strptime('Sep 11 00:00:00', '%b %d %H:%M:%S'))},
+        {'msg' => '[Sep  1 00:00:00] localhost logger: foo', 'expected' => Fluent::EventTime.from_time(Time.strptime('Sep  1 00:00:00', '%b  %d %H:%M:%S'))},
       ]
 
       d.run do
@@ -64,14 +67,16 @@ class UdpInputTest < Test::Unit::TestCase
         tests.each {|test|
           u.send(test['msg'], 0)
         }
+        u.close
         sleep 1
       end
 
       emits = d.emits
       emits.each_index {|i|
-        assert_equal(tests[i]['expected'], emits[i][1])
+        assert_equal_event_time(tests[i]['expected'], emits[i][1])
       }
     }
+
   end
 
   {
@@ -98,6 +103,7 @@ class UdpInputTest < Test::Unit::TestCase
         tests.each { |test|
           u.send(test['msg'], 0)
         }
+        u.close
         sleep 1
       end
 
@@ -105,63 +111,11 @@ class UdpInputTest < Test::Unit::TestCase
     end
   }
 
-  test "remove_newline" do
-    tests = [{'msg' => "test1\n", 'expected' => "test1\n"},
-             {'msg' => "test2\n", 'expected' => "test2\n"}]
-
-    d = create_driver(BASE_CONFIG + %!
-      format none
-      remove_newline false
-    !)
-    d.run do
-      u = UDPSocket.new
-      u.connect('127.0.0.1', PORT)
-      tests.each { |test|
-        u.send(test['msg'], 0)
-      }
-      sleep 1
-    end
-
-    compare_test_result(d.emits, tests)
-  end
-
-  test "receive_buffer_size" do
-    # don't check exact value because it depends on platform and condition
-
-    # check if default socket and in_udp's one without receive_buffer_size have same size buffer
-    d0 = create_driver(BASE_CONFIG + %!
-      format none
-    !)
-    d0.run do
-      begin
-        sock = d0.instance.instance_variable_get(:@handler).instance_variable_get(:@io)
-        default_sock = UDPSocket.new
-        assert_equal(default_sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int, sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int)
-      ensure
-        default_sock.close
-      end
-    end
-
-    # check if default socket and in_udp's one with receive_buffer_size have different size buffer
-    d1 = create_driver(BASE_CONFIG + %!
-      format none
-      receive_buffer_size 1001
-    !)
-    d1.run do
-      sock = d1.instance.instance_variable_get(:@handler).instance_variable_get(:@io)
-      begin
-        default_sock = UDPSocket.new
-        assert_not_equal(default_sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int, sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int)
-      ensure
-        default_sock.close
-      end
-    end
-  end
-
   def compare_test_result(emits, tests)
     assert_equal(2, emits.size)
     emits.each_index {|i|
       assert_equal(tests[i]['expected'], emits[i][2]['message'])
+      assert(emits[i][1].is_a?(Fluent::EventTime))
     }
   end
 end

data/test/plugin/test_input.rb

@@ -0,0 +1,122 @@
+require_relative '../helper'
+require 'fluent/plugin/input'
+require 'flexmock/test_unit'
+
+module FluentPluginInputTest
+  class DummyPlugin < Fluent::Plugin::Input
+  end
+end
+
+class InputTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+    @p = FluentPluginInputTest::DummyPlugin.new
+  end
+
+  test 'has healthy lifecycle' do
+    assert !@p.configured?
+    @p.configure(config_element())
+    assert @p.configured?
+
+    assert !@p.started?
+    @p.start
+    assert @p.start
+
+    assert !@p.stopped?
+    @p.stop
+    assert @p.stopped?
+
+    assert !@p.before_shutdown?
+    @p.before_shutdown
+    assert @p.before_shutdown?
+
+    assert !@p.shutdown?
+    @p.shutdown
+    assert @p.shutdown?
+
+    assert !@p.after_shutdown?
+    @p.after_shutdown
+    assert @p.after_shutdown?
+
+    assert !@p.closed?
+    @p.close
+    assert @p.closed?
+
+    assert !@p.terminated?
+    @p.terminate
+    assert @p.terminated?
+  end
+
+  test 'has plugin_id automatically generated' do
+    assert @p.respond_to?(:plugin_id_configured?)
+    assert @p.respond_to?(:plugin_id)
+
+    @p.configure(config_element())
+
+    assert !@p.plugin_id_configured?
+    assert @p.plugin_id
+    assert{ @p.plugin_id != 'mytest' }
+  end
+
+  test 'has plugin_id manually configured' do
+    @p.configure(config_element('ROOT', '', {'@id' => 'mytest'}))
+    assert @p.plugin_id_configured?
+    assert_equal 'mytest', @p.plugin_id
+  end
+
+  test 'has plugin logger' do
+    assert @p.respond_to?(:log)
+    assert @p.log
+
+    # default logger
+    original_logger = @p.log
+
+    @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
+
+    assert{ @p.log.object_id != original_logger.object_id }
+    assert_equal Fluent::Log::LEVEL_DEBUG, @p.log.level
+  end
+
+  test 'can load plugin helpers' do
+    assert_nothing_raised do
+      class FluentPluginInputTest::DummyPlugin2 < Fluent::Plugin::Input
+        helpers :storage
+      end
+    end
+  end
+
+  test 'has router and can emit into it' do
+    assert @p.has_router?
+
+    @p.configure(config_element())
+    assert @p.router
+
+    DummyRouter = Struct.new(:emits) do
+      def emit(tag, es)
+        self.emits << [tag, es]
+      end
+    end
+    @p.router = DummyRouter.new([])
+    @p.router.emit('mytag', [])
+    @p.router.emit('mytag.testing', ['it is not es, but no problem for tests'])
+
+    assert_equal ['mytag', []], @p.router.emits[0]
+    assert_equal ['mytag.testing', ['it is not es, but no problem for tests']], @p.router.emits[1]
+  end
+
+  test 'has router for specified label if configured' do
+    @p.configure(config_element())
+    original_router = @p.router
+
+    router_mock = flexmock('mytest')
+    router_mock.should_receive(:emit).once.with('mytag.testing', ['for mock'])
+    label_mock = flexmock('mylabel')
+    label_mock.should_receive(:event_router).once.and_return(router_mock)
+    Fluent::Engine.root_agent.labels['@mytest'] = label_mock
+
+    @p.configure(config_element('ROOT', '', {'@label' => '@mytest'}))
+    assert{ @p.router.object_id != original_router.object_id }
+
+    @p.router.emit('mytag.testing', ['for mock'])
+  end
+end

data/test/plugin/test_multi_output.rb

@@ -0,0 +1,180 @@
+require_relative '../helper'
+require 'fluent/plugin/multi_output'
+require 'fluent/event'
+
+require 'json'
+require 'time'
+require 'timeout'
+
+module FluentPluginMultiOutputTest
+  class DummyMultiOutput < Fluent::Plugin::MultiOutput
+    attr_reader :events
+    def initialize
+      super
+      @events = []
+    end
+    def configure(conf)
+      super
+    end
+    def process(tag, es)
+      es.each do |time, record|
+        @events << [tag, time, record]
+      end
+    end
+  end
+  class DummyCompatMultiOutput < Fluent::Plugin::MultiOutput
+    def initialize
+      super
+      @compat = true
+    end
+    def configure(conf)
+      super
+    end
+    def process(tag, es)
+      # ...
+    end
+  end
+
+  class Dummy1Output < Fluent::Plugin::Output
+    Fluent::Plugin.register_output('dummy_test_multi_output_1', self)
+    attr_reader :configured
+    def configure(conf)
+      super
+      @configured = true
+    end
+    def process(tag, es)
+    end
+  end
+  class Dummy2Output < Fluent::Plugin::Output
+    Fluent::Plugin.register_output('dummy_test_multi_output_2', self)
+    attr_reader :configured
+    def configure(conf)
+      super
+      @configured = true
+    end
+    def process(tag, es)
+    end
+  end
+  class Dummy3Output < Fluent::Plugin::Output
+    Fluent::Plugin.register_output('dummy_test_multi_output_3', self)
+    attr_reader :configured
+    def configure(conf)
+      super
+      @configured = true
+    end
+    def process(tag, es)
+    end
+  end
+  class Dummy4Output < Fluent::Plugin::Output
+    Fluent::Plugin.register_output('dummy_test_multi_output_4', self)
+    attr_reader :configured
+    def configure(conf)
+      super
+      @configured = true
+    end
+    def process(tag, es)
+    end
+  end
+end
+
+class MultiOutputTest < Test::Unit::TestCase
+  def create_output(type=:multi)
+    case type
+    when :compat_multi
+      FluentPluginMultiOutputTest::DummyCompatMultiOutput.new
+    else
+      FluentPluginMultiOutputTest::DummyMultiOutput.new
+    end
+  end
+
+  sub_test_case 'basic multi output plugin' do
+    setup do
+      Fluent::Test.setup
+      @i = create_output()
+    end
+
+    teardown do
+      @i.log.out.reset
+    end
+
+    test '#configure raises error if <store> sections are missing' do
+      conf = config_element('ROOT', '', { '@type' => 'dummy_test_multi_output' }, [])
+      assert_raise Fluent::ConfigError do
+        @i.configure(conf)
+      end
+    end
+
+    test '#configure initialize child plugins and call these #configure' do
+      assert_equal [], @i.outputs
+
+      conf = config_element('ROOT', '', { '@type' => 'dummy_test_multi_output' },
+        [
+          config_element('store', '', { '@type' => 'dummy_test_multi_output_1' }),
+          config_element('store', '', { '@type' => 'dummy_test_multi_output_2' }),
+          config_element('store', '', { '@type' => 'dummy_test_multi_output_3' }),
+          config_element('store', '', { '@type' => 'dummy_test_multi_output_4' }),
+        ]
+      )
+      @i.configure(conf)
+
+      assert_equal 4, @i.outputs.size
+
+      assert @i.outputs[0].is_a? FluentPluginMultiOutputTest::Dummy1Output
+      assert @i.outputs[0].configured
+
+      assert @i.outputs[1].is_a? FluentPluginMultiOutputTest::Dummy2Output
+      assert @i.outputs[1].configured
+
+      assert @i.outputs[2].is_a? FluentPluginMultiOutputTest::Dummy3Output
+      assert @i.outputs[2].configured
+
+      assert @i.outputs[3].is_a? FluentPluginMultiOutputTest::Dummy4Output
+      assert @i.outputs[3].configured
+    end
+
+    test '#configure warns if "type" is used in <store> sections instead of "@type"' do
+      assert_equal [], @i.log.out.logs
+
+      conf = config_element('ROOT', '', { '@type' => 'dummy_test_multi_output' },
+        [
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_1' }),
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_2' }),
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_3' }),
+          config_element('store', '', { 'type' => 'dummy_test_multi_output_4' }),
+        ]
+      )
+      @i.configure(conf)
+      assert_equal 4, @i.outputs.size
+
+      logs = @i.log.out.logs
+      assert{ logs.select{|log| log.include?('[warn]') && log.include?("'type' is deprecated parameter name. use '@type' instead.") }.size == 4 }
+    end
+
+    test '#emit_events calls #process always' do
+      conf = config_element('ROOT', '', { '@type' => 'dummy_test_multi_output' },
+        [
+          config_element('store', '', { '@type' => 'dummy_test_multi_output_1' }),
+          config_element('store', '', { '@type' => 'dummy_test_multi_output_2' }),
+          config_element('store', '', { '@type' => 'dummy_test_multi_output_3' }),
+          config_element('store', '', { '@type' => 'dummy_test_multi_output_4' }),
+        ]
+      )
+      @i.configure(conf)
+      @i.start
+
+      assert @i.events.empty?
+
+      @i.emit_events(
+        'test.tag',
+        Fluent::ArrayEventStream.new(
+          [
+            [event_time(), {"message" => "multi test 1"}],
+            [event_time(), {"message" => "multi test 1"}],
+          ]
+        )
+      )
+
+      assert_equal 2, @i.events.size
+    end
+  end
+end