fluentd 0.12.40 → 1.6.2

data/test/plugin/test_in_object_space.rb

@@ -1,7 +1,21 @@
 require_relative '../helper'
-require 'fluent/test'
+require 'fluent/test/driver/input'
+require 'fluent/plugin/in_object_space'
+
+require 'timeout'
 
 class ObjectSpaceInputTest < Test::Unit::TestCase
+  def waiting(seconds, instance)
+    begin
+      Timeout.timeout(seconds) do
+        yield
+      end
+    rescue Timeout::Error
+      STDERR.print(*instance.log.out.logs)
+      raise
+    end
+  end
+
   class FailObject
     def self.class
       raise "error"
@@ -19,7 +33,7 @@ class ObjectSpaceInputTest < Test::Unit::TestCase
   ]
 
   def create_driver(conf=TESTCONFIG)
-    Fluent::Test::InputTestDriver.new(Fluent::ObjectSpaceInput).configure(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::ObjectSpaceInput).configure(conf)
   end
 
   def test_configure
@@ -32,17 +46,19 @@ class ObjectSpaceInputTest < Test::Unit::TestCase
   def test_emit
     d = create_driver
 
-    time = Time.parse("2011-01-02 13:14:15").to_i
-
-    d.expected_emits_length = 2
-    d.run
+    d.run do
+      waiting(10, d.instance) do
+        sleep 0.5 until d.events.size > 3
+      end
+    end
 
-    emits = d.emits
-    assert_equal true, emits.length > 0
+    emits = d.events
+    assert{ emits.length > 0 }
 
     emits.each { |tag, time, record|
       assert_equal d.instance.tag, tag
       assert_equal d.instance.top, record.keys.size
+      assert(time.is_a?(Fluent::EventTime))
     }
   end
 end

data/test/plugin/test_in_syslog.rb

@@ -1,10 +1,10 @@
 require_relative '../helper'
-require 'fluent/test'
+require 'fluent/test/driver/input'
+require 'fluent/plugin/in_syslog'
 
 class SyslogInputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
-    require 'fluent/plugin/socket_util'
   end
 
   PORT = unused_port
@@ -21,18 +21,20 @@ class SyslogInputTest < Test::Unit::TestCase
   ]
 
   def create_driver(conf=CONFIG)
-    Fluent::Test::InputTestDriver.new(Fluent::SyslogInput).configure(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::SyslogInput).configure(conf)
   end
 
-  def test_configure
-    configs = {'127.0.0.1' => CONFIG}
-    configs.merge!('::1' => IPv6_CONFIG) if ipv6_enabled?
-
-    configs.each_pair { |k, v|
-      d = create_driver(v)
-      assert_equal PORT, d.instance.port
-      assert_equal k, d.instance.bind
-    }
+  data(
+    ipv4: ['127.0.0.1', CONFIG, ::Socket::AF_INET],
+    ipv6: ['::1', IPv6_CONFIG, ::Socket::AF_INET6],
+  )
+  def test_configure(data)
+    bind_addr, config, family = data
+    omit "IPv6 unavailable" if family == ::Socket::AF_INET6 && !ipv6_enabled?
+
+    d = create_driver(config)
+    assert_equal PORT, d.instance.port
+    assert_equal bind_addr, d.instance.bind
   end
 
   sub_test_case 'source_hostname_key and source_address_key features' do
@@ -45,90 +47,69 @@ EOS
       }
     end
 
-    LOCALHOST_HOSTNAME_GETTER = ->(){sock = UDPSocket.new(::Socket::AF_INET); sock.do_not_reverse_lookup = false; sock.connect("127.0.0.1", 2048); sock.peeraddr[2] }
-    LOCALHOST_HOSTNAME = LOCALHOST_HOSTNAME_GETTER.call
-    DUMMY_SOCK = Struct.new(:remote_host, :remote_addr, :remote_port).new(LOCALHOST_HOSTNAME, "127.0.0.1", 0)
-    data(
-      both: [:hostname, :address],
-      hostname: [:hostname],
-      address: [:address],
-    )
-    test 'source_hostname_key and source_address_key parameter feature should add record(s)' do |keys|
-      conf = CONFIG.dup
-      if keys.include?(:hostname)
-        conf << <<EOL
-source_hostname_key source_hostname
-EOL
-        end
-      if keys.include?(:address)
-        conf << <<EOL
-source_address_key source_address
-EOL
-      end
-      tests = create_test_case
-      d = create_driver(conf)
-
-      d.run do
-        u = UDPSocket.new
-        u.connect('127.0.0.1', PORT)
-        tests.each {|test|
-          u.send(test['msg'], 0)
-        }
-        sleep 1
-      end
-
-      d.emits.each { |tag, _time, record|
-        if keys.include?(:hostname)
-          assert_true record.has_key?('source_hostname')
-          assert_equal DUMMY_SOCK.remote_host, record['source_hostname']
-          unless keys.include?(:address)
-            assert_false record.has_key?('source_address')
-          end
-        end
-        if keys.include?(:address)
-          assert_true record.has_key?('source_address')
-          assert_equal DUMMY_SOCK.remote_addr, record['source_address']
-          unless keys.include?(:hostname)
-            assert_false record.has_key?('source_hostname')
-          end
-        end
-      }
-    end
-
     data('resolve_hostname' => 'resolve_hostname true',
          'source_hostname_key' => 'source_hostname_key source_host')
-    def test_configure_reslove_hostname(param)
+    def test_configure_resolve_hostname(param)
       d = create_driver([CONFIG, param].join("\n"))
       assert_true d.instance.resolve_hostname
     end
   end
 
-  def test_time_format
-    configs = {'127.0.0.1' => CONFIG}
-    configs.merge!('::1' => IPv6_CONFIG) if ipv6_enabled?
+  data('Use protocol_type' => ['protocol_type tcp', :tcp, :udp],
+       'Use transport' => ["<transport tcp>\n </transport>", nil, :tcp],
+       'Use transport and protocol' => ["protocol_type udp\n<transport tcp>\n </transport>", :udp, :tcp])
+  def test_configure_protocol(param)
+    conf, proto_type, transport_proto_type = *param
+    d = create_driver([CONFIG, conf].join("\n"))
 
-    configs.each_pair { |k, v|
-      d = create_driver(v)
+    assert_equal(d.instance.protocol_type, proto_type)
+    assert_equal(d.instance.transport_config.protocol, transport_proto_type)
+  end
 
-      tests = [
-        {'msg' => '<6>Sep 11 00:00:00 localhost logger: foo', 'expected' => Time.strptime('Sep 11 00:00:00', '%b %d %H:%M:%S').to_i},
-        {'msg' => '<6>Sep  1 00:00:00 localhost logger: foo', 'expected' => Time.strptime('Sep  1 00:00:00', '%b  %d %H:%M:%S').to_i},
-      ]
+  # For backward compat
+  def test_respect_protocol_type_than_transport
+    d = create_driver([CONFIG, "<transport tcp> \n</transport>", "protocol_type udp"].join("\n"))
+    tests = create_test_case
 
-      d.run do
-        u = Fluent::SocketUtil.create_udp_socket(k)
-        u.connect(k, PORT)
-        tests.each {|test|
-          u.send(test['msg'], 0)
-        }
-        sleep 1
-      end
+    d.run(expect_emits: 2) do
+      u = UDPSocket.new
+      u.connect('127.0.0.1', PORT)
+      tests.each {|test|
+        u.send(test['msg'], 0)
+      }
+    end
+
+    assert(d.events.size > 0)
+    compare_test_result(d.events, tests)
+  end
+
+
+  data(
+    ipv4: ['127.0.0.1', CONFIG, ::Socket::AF_INET],
+    ipv6: ['::1', IPv6_CONFIG, ::Socket::AF_INET6],
+  )
+  def test_time_format(data)
+    bind_addr, config, family = data
+    omit "IPv6 unavailable" if family == ::Socket::AF_INET6 && !ipv6_enabled?
 
-      emits = d.emits
-      assert_equal 2, emits.size
-      emits.each_index {|i|
-        assert_equal(tests[i]['expected'], emits[i][1])
+    d = create_driver(config)
+
+    tests = [
+      {'msg' => '<6>Dec 11 00:00:00 localhost logger: foo', 'expected' => Fluent::EventTime.from_time(Time.strptime('Dec 11 00:00:00', '%b %d %H:%M:%S'))},
+      {'msg' => '<6>Dec  1 00:00:00 localhost logger: foo', 'expected' => Fluent::EventTime.from_time(Time.strptime('Dec  1 00:00:00', '%b  %d %H:%M:%S'))},
+    ]
+    d.run(expect_emits: 2) do
+      u = UDPSocket.new(family)
+      u.connect(bind_addr, PORT)
+      tests.each {|test|
+        u.send(test['msg'], 0)
       }
+    end
+
+    events = d.events
+    assert(events.size > 0)
+    events.each_index {|i|
+      assert_equal_event_time(tests[i]['expected'], events[i][1])
     }
   end
 
@@ -136,70 +117,66 @@ EOL
     d = create_driver
     tests = create_test_case
 
-    d.run do
+    d.run(expect_emits: 2) do
       u = UDPSocket.new
       u.connect('127.0.0.1', PORT)
       tests.each {|test|
         u.send(test['msg'], 0)
       }
-      sleep 1
     end
 
-    assert_equal 2, d.emits.size
-    compare_test_result(d.emits, tests)
+    assert(d.events.size > 0)
+    compare_test_result(d.events, tests)
   end
 
   def test_msg_size_udp_for_large_msg
     d = create_driver(CONFIG + %[
       message_length_limit 5k
     ])
-    tests = create_test_case(true)
+    tests = create_test_case(large_message: true)
 
-    d.run do
+    d.run(expect_emits: 3) do
       u = UDPSocket.new
       u.connect('127.0.0.1', PORT)
       tests.each {|test|
         u.send(test['msg'], 0)
       }
-      sleep 1
     end
 
-    assert_equal 3, d.emits.size
-    compare_test_result(d.emits, tests)
+    assert(d.events.size > 0)
+    compare_test_result(d.events, tests)
   end
 
   def test_msg_size_with_tcp
-    d = create_driver([CONFIG, 'protocol_type tcp'].join("\n"))
+    d = create_driver([CONFIG, "<transport tcp> \n</transport>"].join("\n"))
     tests = create_test_case
 
-    d.run do
+    d.run(expect_emits: 2) do
       tests.each {|test|
         TCPSocket.open('127.0.0.1', PORT) do |s|
           s.send(test['msg'], 0)
         end
       }
-      sleep 1
     end
 
-    assert_equal 2, d.emits.size
-    compare_test_result(d.emits, tests)
+    assert(d.events.size > 0)
+    compare_test_result(d.events, tests)
   end
 
   def test_msg_size_with_same_tcp_connection
-    d = create_driver([CONFIG, 'protocol_type tcp'].join("\n"))
+    d = create_driver([CONFIG, "<transport tcp> \n</transport>"].join("\n"))
     tests = create_test_case
 
-    d.run do
+    d.run(expect_emits: 2) do
       TCPSocket.open('127.0.0.1', PORT) do |s|
         tests.each {|test|
           s.send(test['msg'], 0)
         }
       end
-      sleep 1
     end
 
-    assert_equal 2, d.emits.size
-    compare_test_result(d.emits, tests)
+    assert(d.events.size > 0)
+    compare_test_result(d.events, tests)
   end
 
   def test_msg_size_with_json_format
@@ -210,149 +187,108 @@ EOL
       {'msg' => '<6>' + event.to_json + "\n", 'expected' => msg}
     }
 
-    d.run do
+    d.run(expect_emits: 2) do
       u = UDPSocket.new
       u.connect('127.0.0.1', PORT)
       tests.each {|test|
         u.send(test['msg'], 0)
       }
-      sleep 1
     end
 
-    assert_equal 2, d.emits.size
-    compare_test_result(d.emits, tests)
+    assert(d.events.size > 0)
+    compare_test_result(d.events, tests)
   end
 
-  LOCALHOST_HOSTNAME_GETTER = ->(){sock = UDPSocket.new(::Socket::AF_INET); sock.do_not_reverse_lookup = false; sock.connect("127.0.0.1", 2048); sock.peeraddr[2] }
-  LOCALHOST_HOSTNAME = LOCALHOST_HOSTNAME_GETTER.call
-
-  data('old parameter' => 'include_source_host',
-       'new parameter' => 'source_hostname_key source_host')
-  def test_msg_size_with_include_source_host(param)
-    d = create_driver([CONFIG, param].join("\n"))
+  def test_msg_size_with_include_source_host
+    d = create_driver([CONFIG, 'include_source_host true'].join("\n"))
     tests = create_test_case
 
-    d.run do
+    host = nil
+    d.run(expect_emits: 2) do
       u = UDPSocket.new
       u.connect('127.0.0.1', PORT)
+      host = u.peeraddr[2]
       tests.each {|test|
         u.send(test['msg'], 0)
       }
-      sleep 1
     end
 
-    assert_equal 2, d.emits.size
-    compare_test_result(d.emits, tests, {host: LOCALHOST_HOSTNAME})
+    assert(d.events.size > 0)
+    compare_test_result(d.events, tests, {host: host})
   end
 
-  def test_msg_size_with_include_priority
+  def test_msg_size_with_priority_key
     d = create_driver([CONFIG, 'priority_key priority'].join("\n"))
     tests = create_test_case
 
     priority = 'info'
-    d.run do
+    d.run(expect_emits: 2) do
       u = UDPSocket.new
       u.connect('127.0.0.1', PORT)
       tests.each {|test|
         u.send(test['msg'], 0)
       }
-      sleep 1
     end
 
-    assert_equal 2, d.emits.size
-    compare_test_result(d.emits, tests, {priority: priority})
+    assert(d.events.size > 0)
+    compare_test_result(d.events, tests, {priority: priority})
   end
 
-  def test_msg_size_with_include_facility
+  def test_msg_size_with_facility_key
     d = create_driver([CONFIG, 'facility_key facility'].join("\n"))
     tests = create_test_case
 
     facility = 'kern'
-    d.run do
-      u = UDPSocket.new
-      u.connect('127.0.0.1', PORT)
-      tests.each {|test|
-        u.send(test['msg'], 0)
-      }
-      sleep 1
-    end
-
-    assert_equal 2, d.emits.size
-    compare_test_result(d.emits, tests, {facility: facility})
-  end
-
-  def test_allow_without_priority_with_default_format
-    d = create_driver([CONFIG, 'allow_without_priority true'].join("\n"))
-
-    tests = [
-      {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100 + "\n", 'expected' => 'x' * 100},
-      {'msg' => 'Sep 10 00:00:00 localhost logger: ' + 'x' * 1024 + "\n", 'expected' => 'x' * 1024, 'tag' => 'syslog.user.notice'},
-    ]
-
-    d.run do
+    d.run(expect_emits: 2) do
       u = UDPSocket.new
       u.connect('127.0.0.1', PORT)
       tests.each {|test|
         u.send(test['msg'], 0)
       }
-      sleep 1
     end
 
-    assert_equal 2, d.emits.size
-    compare_test_result(d.emits, tests)
+    assert(d.events.size > 0)
+    compare_test_result(d.events, tests, {facility: facility})
   end
 
-  def test_allow_without_priority_with_json_format
-    d = create_driver([CONFIG, %[
-      allow_without_priority true
-      format json
-    ]].join("\n"))
-
-    message = 'foo'
-    tests = [
-      {'msg' => '<6>' + {'message' => message}.to_json + "\n", 'expected' => message},
-      {'msg' => {'message' => message}.to_json + "\n", 'expected' => message, 'tag' => 'syslog.user.notice'},
-    ]
+  def test_msg_size_with_source_address_key
+    d = create_driver([CONFIG, 'source_address_key source_address'].join("\n"))
+    tests = create_test_case
 
-    d.run do
+    address = nil
+    d.run(expect_emits: 2) do
       u = UDPSocket.new
       u.connect('127.0.0.1', PORT)
+      address = u.peeraddr[3]
       tests.each {|test|
         u.send(test['msg'], 0)
       }
-      sleep 1
     end
 
-    assert_equal 2, d.emits.size
-    compare_test_result(d.emits, tests)
+    assert(d.events.size > 0)
+    compare_test_result(d.events, tests, {address: address})
   end
 
-  def test_default_priority
-    d = create_driver([CONFIG, %[
-      allow_without_priority true
-      default_priority 100
-    ]].join("\n"))
-
-    tests = [
-      {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100 + "\n", 'expected' => 'x' * 100},
-      {'msg' => 'Sep 10 00:00:00 localhost logger: ' + 'x' * 1024 + "\n", 'expected' => 'x' * 1024, 'tag' => 'syslog.ntp.warn'},
-    ]
+  def test_msg_size_with_source_hostname_key
+    d = create_driver([CONFIG, 'source_hostname_key source_hostname'].join("\n"))
+    tests = create_test_case
 
-    d.run do
+    hostname = nil
+    d.run(expect_emits: 2) do
       u = UDPSocket.new
+      u.do_not_reverse_lookup = false
       u.connect('127.0.0.1', PORT)
+      hostname = u.peeraddr[2]
       tests.each {|test|
         u.send(test['msg'], 0)
       }
-      sleep 1
     end
 
-    assert_equal 2, d.emits.size
-    compare_test_result(d.emits, tests)
-
+    assert(d.events.size > 0)
+    compare_test_result(d.events, tests, {hostname: hostname})
   end
 
-  def create_test_case(large_message = false)
+  def create_test_case(large_message: false)
     # actual syslog message has "\n"
     if large_message
       [
@@ -370,56 +306,59 @@ EOL
 
   def compare_test_result(events, tests, options = {})
     events.each_index { |i|
-      assert_equal((tests[i]['tag'] || 'syslog.kern.info'), events[i][0]) # <6> means kern.info
+      assert_equal('syslog.kern.info', events[i][0]) # <6> means kern.info
       assert_equal(tests[i]['expected'], events[i][2]['message'])
       assert_equal(options[:host], events[i][2]['source_host']) if options[:host]
+      assert_equal(options[:address], events[i][2]['source_address']) if options[:address]
+      assert_equal(options[:hostname], events[i][2]['source_hostname']) if options[:hostname]
       assert_equal(options[:priority], events[i][2]['priority']) if options[:priority]
       assert_equal(options[:facility], events[i][2]['facility']) if options[:facility]
     }
   end
 
-  class SyslogMessageFormatTest < self
-    def test_syslog_rfc5424_format
-      d = create_driver(CONFIG + 'message_format rfc5424')
-      tests = [
-        '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!',
-        '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
-      ]
-
-      run_tests(d, tests)
-      compare_test_result(d)
-    end
+  sub_test_case 'octet counting frame' do
+    def test_msg_size_with_tcp
+      d = create_driver([CONFIG, "<transport tcp> \n</transport>", 'frame_type octet_count'].join("\n"))
+      tests = create_test_case
 
-    def test_syslog_auto_format
-      d = create_driver(CONFIG + 'message_format auto')
-      tests = [
-        '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!',
-        '<6>Sep 11 00:00:00 localhost fluentd: Hi, from Fluentd!'
-      ]
+      d.run(expect_emits: 2) do
+        tests.each {|test|
+          TCPSocket.open('127.0.0.1', PORT) do |s|
+            s.send(test['msg'], 0)
+          end
+        }
+      end
 
-      run_tests(d, tests)
-      compare_test_result(d)
+      assert(d.events.size > 0)
+      compare_test_result(d.events, tests)
     end
 
-    def run_tests(d, tests)
-      d.run do
-        u = Fluent::SocketUtil.create_udp_socket('127.0.0.1')
-        u.connect('127.0.0.1', PORT)
-        tests.each {|test|
-          u.send(test, 0)
-        }
-        sleep 1
+    def test_msg_size_with_same_tcp_connection
+      d = create_driver([CONFIG, "<transport tcp> \n</transport>", 'frame_type octet_count'].join("\n"))
+      tests = create_test_case
+
+      d.run(expect_emits: 2) do
+        TCPSocket.open('127.0.0.1', PORT) do |s|
+          tests.each {|test|
+            s.send(test['msg'], 0)
+          }
+        end
       end
+
+      assert(d.events.size > 0)
+      compare_test_result(d.events, tests)
     end
 
-    def compare_test_result(d)
-      emits = d.emits
-      assert_equal 2, emits.size
-      emits.each_index {|i|
-        record =  emits[i][2]
-        assert_equal('fluentd', record['ident'])
-        assert_equal('Hi, from Fluentd!', record['message'])
+    def create_test_case(large_message: false)
+      msgs = [
+        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100, 'expected' => 'x' * 100},
+        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 1024, 'expected' => 'x' * 1024},
+      ]
+      msgs.each { |msg|
+        m = msg['msg']
+        msg['msg'] = "#{m.size + 1} #{m}"
       }
+      msgs
     end
   end
 end