fluentd 0.12.40 → 0.14.0

data/test/plugin/test_in_object_space.rb

@@ -1,7 +1,21 @@
 require_relative '../helper'
 require 'fluent/test'
+require 'fluent/plugin/in_object_space'
+
+require 'timeout'
 
 class ObjectSpaceInputTest < Test::Unit::TestCase
+  def waiting(seconds, instance)
+    begin
+      Timeout.timeout(seconds) do
+        yield
+      end
+    rescue Timeout::Error
+      STDERR.print(*instance.log.out.logs)
+      raise
+    end
+  end
+
   class FailObject
     def self.class
       raise "error"
@@ -32,17 +46,19 @@ class ObjectSpaceInputTest < Test::Unit::TestCase
   def test_emit
     d = create_driver
 
-    time = Time.parse("2011-01-02 13:14:15").to_i
-
-    d.expected_emits_length = 2
-    d.run
+    d.run do
+      waiting(10, d.instance) do
+        sleep 0.5 until d.emit_streams.size > 3
+      end
+    end
 
     emits = d.emits
-    assert_equal true, emits.length > 0
+    assert{ emits.length > 0 }
 
     emits.each { |tag, time, record|
       assert_equal d.instance.tag, tag
       assert_equal d.instance.top, record.keys.size
+      assert(time.is_a?(Fluent::EventTime))
     }
   end
 end

data/test/plugin/test_in_stream.rb

@@ -1,5 +1,6 @@
 require_relative '../helper'
 require 'fluent/test'
+require 'fluent/plugin/in_stream'
 
 module StreamInputTest
   def setup
@@ -9,14 +10,14 @@ module StreamInputTest
   def test_time
     d = create_driver
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
     Fluent::Engine.now = time
 
     d.expect_emit "tag1", time, {"a"=>1}
     d.expect_emit "tag2", time, {"a"=>2}
 
     d.run do
-      d.expected_emits.each {|tag,time,record|
+      d.expected_emits.each {|tag,_time,record|
         send_data Fluent::Engine.msgpack_factory.packer.write([tag, 0, record]).to_s
       }
     end
@@ -25,14 +26,14 @@ module StreamInputTest
   def test_message
     d = create_driver
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
 
     d.expect_emit "tag1", time, {"a"=>1}
     d.expect_emit "tag2", time, {"a"=>2}
 
     d.run do
-      d.expected_emits.each {|tag,time,record|
-        send_data Fluent::Engine.msgpack_factory.packer.write([tag, time, record]).to_s
+      d.expected_emits.each {|tag,_time,record|
+        send_data Fluent::Engine.msgpack_factory.packer.write([tag, _time, record]).to_s
       }
     end
   end
@@ -40,15 +41,15 @@ module StreamInputTest
   def test_forward
     d = create_driver
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
 
     d.expect_emit "tag1", time, {"a"=>1}
     d.expect_emit "tag1", time, {"a"=>2}
 
     d.run do
       entries = []
-      d.expected_emits.each {|tag,time,record|
-        entries << [time, record]
+      d.expected_emits.each {|tag,_time,record|
+        entries << [_time, record]
       }
       send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
     end
@@ -57,15 +58,15 @@ module StreamInputTest
   def test_packed_forward
     d = create_driver
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
 
     d.expect_emit "tag1", time, {"a"=>1}
     d.expect_emit "tag1", time, {"a"=>2}
 
     d.run do
       entries = ''
-      d.expected_emits.each {|tag,time,record|
-        Fluent::Engine.msgpack_factory.packer(entries).write([time, record]).flush
+      d.expected_emits.each {|tag,_time,record|
+        Fluent::Engine.msgpack_factory.packer(entries).write([_time, record]).flush
       }
       send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
     end
@@ -80,7 +81,7 @@ module StreamInputTest
     d.expect_emit "tag2", time, {"a"=>2}
 
     d.run do
-      d.expected_emits.each {|tag,time,record|
+      d.expected_emits.each {|tag,_time,record|
         send_data [tag, time, record].to_json
       }
     end
@@ -122,4 +123,4 @@ class UnixInputTest < Test::Unit::TestCase
   def connect
     UNIXSocket.new("#{TMP_DIR}/unix")
   end
-end
+end unless Fluent.windows?

data/test/plugin/test_in_syslog.rb

@@ -1,7 +1,20 @@
 require_relative '../helper'
 require 'fluent/test'
+require 'fluent/plugin/in_syslog'
 
 class SyslogInputTest < Test::Unit::TestCase
+  class << self
+    def startup
+      socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+      @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
+      ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+    end
+
+    def shutdown
+      @server.close
+    end
+  end
+  
   def setup
     Fluent::Test.setup
     require 'fluent/plugin/socket_util'
@@ -35,74 +48,6 @@ class SyslogInputTest < Test::Unit::TestCase
     }
   end
 
-  sub_test_case 'source_hostname_key and source_address_key features' do
-    test 'resolve_hostname must be true with source_hostname_key' do
-      assert_raise(Fluent::ConfigError) {
-        create_driver(CONFIG + <<EOS)
-resolve_hostname false
-source_hostname_key hostname
-EOS
-      }
-    end
-
-    LOCALHOST_HOSTNAME_GETTER = ->(){sock = UDPSocket.new(::Socket::AF_INET); sock.do_not_reverse_lookup = false; sock.connect("127.0.0.1", 2048); sock.peeraddr[2] }
-    LOCALHOST_HOSTNAME = LOCALHOST_HOSTNAME_GETTER.call
-    DUMMY_SOCK = Struct.new(:remote_host, :remote_addr, :remote_port).new(LOCALHOST_HOSTNAME, "127.0.0.1", 0)
-    data(
-      both: [:hostname, :address],
-      hostname: [:hostname],
-      address: [:address],
-    )
-    test 'source_hostname_key and source_address_key parameter feature should add record(s)' do |keys|
-      conf = CONFIG.dup
-      if keys.include?(:hostname)
-        conf << <<EOL
-source_hostname_key source_hostname
-EOL
-        end
-      if keys.include?(:address)
-        conf << <<EOL
-source_address_key source_address
-EOL
-      end
-      tests = create_test_case
-      d = create_driver(conf)
-
-      d.run do
-        u = UDPSocket.new
-        u.connect('127.0.0.1', PORT)
-        tests.each {|test|
-          u.send(test['msg'], 0)
-        }
-        sleep 1
-      end
-
-      d.emits.each { |tag, _time, record|
-        if keys.include?(:hostname)
-          assert_true record.has_key?('source_hostname')
-          assert_equal DUMMY_SOCK.remote_host, record['source_hostname']
-          unless keys.include?(:address)
-            assert_false record.has_key?('source_address')
-          end
-        end
-        if keys.include?(:address)
-          assert_true record.has_key?('source_address')
-          assert_equal DUMMY_SOCK.remote_addr, record['source_address']
-          unless keys.include?(:hostname)
-            assert_false record.has_key?('source_hostname')
-          end
-        end
-      }
-    end
-
-    data('resolve_hostname' => 'resolve_hostname true',
-         'source_hostname_key' => 'source_hostname_key source_host')
-    def test_configure_reslove_hostname(param)
-      d = create_driver([CONFIG, param].join("\n"))
-      assert_true d.instance.resolve_hostname
-    end
-  end
-
   def test_time_format
     configs = {'127.0.0.1' => CONFIG}
     configs.merge!('::1' => IPv6_CONFIG) if ipv6_enabled?
@@ -111,8 +56,8 @@ EOL
       d = create_driver(v)
 
       tests = [
-        {'msg' => '<6>Sep 11 00:00:00 localhost logger: foo', 'expected' => Time.strptime('Sep 11 00:00:00', '%b %d %H:%M:%S').to_i},
-        {'msg' => '<6>Sep  1 00:00:00 localhost logger: foo', 'expected' => Time.strptime('Sep  1 00:00:00', '%b  %d %H:%M:%S').to_i},
+        {'msg' => '<6>Dec 11 00:00:00 localhost logger: foo', 'expected' => Fluent::EventTime.from_time(Time.strptime('Dec 11 00:00:00', '%b %d %H:%M:%S'))},
+        {'msg' => '<6>Dec  1 00:00:00 localhost logger: foo', 'expected' => Fluent::EventTime.from_time(Time.strptime('Dec  1 00:00:00', '%b  %d %H:%M:%S'))},
       ]
 
       d.run do
@@ -125,9 +70,8 @@ EOL
       end
 
       emits = d.emits
-      assert_equal 2, emits.size
       emits.each_index {|i|
-        assert_equal(tests[i]['expected'], emits[i][1])
+        assert_equal_event_time(tests[i]['expected'], emits[i][1])
       }
     }
   end
@@ -145,26 +89,6 @@ EOL
       sleep 1
     end
 
-    assert_equal 2, d.emits.size
-    compare_test_result(d.emits, tests)
-  end
-
-  def test_msg_size_udp_for_large_msg
-    d = create_driver(CONFIG + %[
-      message_length_limit 5k
-    ])
-    tests = create_test_case(true)
-
-    d.run do
-      u = UDPSocket.new
-      u.connect('127.0.0.1', PORT)
-      tests.each {|test|
-        u.send(test['msg'], 0)
-      }
-      sleep 1
-    end
-
-    assert_equal 3, d.emits.size
     compare_test_result(d.emits, tests)
   end
 
@@ -181,7 +105,6 @@ EOL
       sleep 1
     end
 
-    assert_equal 2, d.emits.size
     compare_test_result(d.emits, tests)
   end
 
@@ -198,7 +121,6 @@ EOL
       sleep 1
     end
 
-    assert_equal 2, d.emits.size
     compare_test_result(d.emits, tests)
   end
 
@@ -219,207 +141,40 @@ EOL
       sleep 1
     end
 
-    assert_equal 2, d.emits.size
     compare_test_result(d.emits, tests)
   end
 
-  LOCALHOST_HOSTNAME_GETTER = ->(){sock = UDPSocket.new(::Socket::AF_INET); sock.do_not_reverse_lookup = false; sock.connect("127.0.0.1", 2048); sock.peeraddr[2] }
-  LOCALHOST_HOSTNAME = LOCALHOST_HOSTNAME_GETTER.call
-
-  data('old parameter' => 'include_source_host',
-       'new parameter' => 'source_hostname_key source_host')
-  def test_msg_size_with_include_source_host(param)
-    d = create_driver([CONFIG, param].join("\n"))
-    tests = create_test_case
-
-    d.run do
-      u = UDPSocket.new
-      u.connect('127.0.0.1', PORT)
-      tests.each {|test|
-        u.send(test['msg'], 0)
-      }
-      sleep 1
-    end
-
-    assert_equal 2, d.emits.size
-    compare_test_result(d.emits, tests, {host: LOCALHOST_HOSTNAME})
-  end
-
-  def test_msg_size_with_include_priority
-    d = create_driver([CONFIG, 'priority_key priority'].join("\n"))
+  def test_msg_size_with_include_source_host
+    d = create_driver([CONFIG, 'include_source_host'].join("\n"))
     tests = create_test_case
 
-    priority = 'info'
+    host = nil
     d.run do
       u = UDPSocket.new
       u.connect('127.0.0.1', PORT)
+      host = u.peeraddr[2]
       tests.each {|test|
         u.send(test['msg'], 0)
       }
       sleep 1
     end
 
-    assert_equal 2, d.emits.size
-    compare_test_result(d.emits, tests, {priority: priority})
+    compare_test_result(d.emits, tests, host)
   end
 
-  def test_msg_size_with_include_facility
-    d = create_driver([CONFIG, 'facility_key facility'].join("\n"))
-    tests = create_test_case
-
-    facility = 'kern'
-    d.run do
-      u = UDPSocket.new
-      u.connect('127.0.0.1', PORT)
-      tests.each {|test|
-        u.send(test['msg'], 0)
-      }
-      sleep 1
-    end
-
-    assert_equal 2, d.emits.size
-    compare_test_result(d.emits, tests, {facility: facility})
-  end
-
-  def test_allow_without_priority_with_default_format
-    d = create_driver([CONFIG, 'allow_without_priority true'].join("\n"))
-
-    tests = [
-      {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100 + "\n", 'expected' => 'x' * 100},
-      {'msg' => 'Sep 10 00:00:00 localhost logger: ' + 'x' * 1024 + "\n", 'expected' => 'x' * 1024, 'tag' => 'syslog.user.notice'},
-    ]
-
-    d.run do
-      u = UDPSocket.new
-      u.connect('127.0.0.1', PORT)
-      tests.each {|test|
-        u.send(test['msg'], 0)
-      }
-      sleep 1
-    end
-
-    assert_equal 2, d.emits.size
-    compare_test_result(d.emits, tests)
-  end
-
-  def test_allow_without_priority_with_json_format
-    d = create_driver([CONFIG, %[
-      allow_without_priority true
-      format json
-    ]].join("\n"))
-
-    message = 'foo'
-    tests = [
-      {'msg' => '<6>' + {'message' => message}.to_json + "\n", 'expected' => message},
-      {'msg' => {'message' => message}.to_json + "\n", 'expected' => message, 'tag' => 'syslog.user.notice'},
-    ]
-
-    d.run do
-      u = UDPSocket.new
-      u.connect('127.0.0.1', PORT)
-      tests.each {|test|
-        u.send(test['msg'], 0)
-      }
-      sleep 1
-    end
-
-    assert_equal 2, d.emits.size
-    compare_test_result(d.emits, tests)
-  end
-
-  def test_default_priority
-    d = create_driver([CONFIG, %[
-      allow_without_priority true
-      default_priority 100
-    ]].join("\n"))
-
-    tests = [
+  def create_test_case
+    # actual syslog message has "\n"
+    [
       {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100 + "\n", 'expected' => 'x' * 100},
-      {'msg' => 'Sep 10 00:00:00 localhost logger: ' + 'x' * 1024 + "\n", 'expected' => 'x' * 1024, 'tag' => 'syslog.ntp.warn'},
+      {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 1024 + "\n", 'expected' => 'x' * 1024},
     ]
-
-    d.run do
-      u = UDPSocket.new
-      u.connect('127.0.0.1', PORT)
-      tests.each {|test|
-        u.send(test['msg'], 0)
-      }
-      sleep 1
-    end
-
-    assert_equal 2, d.emits.size
-    compare_test_result(d.emits, tests)
-
   end
 
-  def create_test_case(large_message = false)
-    # actual syslog message has "\n"
-    if large_message
-      [
-        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100 + "\n", 'expected' => 'x' * 100},
-        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 1024 + "\n", 'expected' => 'x' * 1024},
-        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 4096 + "\n", 'expected' => 'x' * 4096},
-      ]
-    else
-      [
-        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100 + "\n", 'expected' => 'x' * 100},
-        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 1024 + "\n", 'expected' => 'x' * 1024},
-      ]
-    end
-  end
-
-  def compare_test_result(events, tests, options = {})
-    events.each_index { |i|
-      assert_equal((tests[i]['tag'] || 'syslog.kern.info'), events[i][0]) # <6> means kern.info
-      assert_equal(tests[i]['expected'], events[i][2]['message'])
-      assert_equal(options[:host], events[i][2]['source_host']) if options[:host]
-      assert_equal(options[:priority], events[i][2]['priority']) if options[:priority]
-      assert_equal(options[:facility], events[i][2]['facility']) if options[:facility]
+  def compare_test_result(emits, tests, host = nil)
+    emits.each_index { |i|
+      assert_equal('syslog.kern.info', emits[0][0]) # <6> means kern.info
+      assert_equal(tests[i]['expected'], emits[i][2]['message'])
+      assert_equal(host, emits[i][2]['source_host']) if host
     }
   end
-
-  class SyslogMessageFormatTest < self
-    def test_syslog_rfc5424_format
-      d = create_driver(CONFIG + 'message_format rfc5424')
-      tests = [
-        '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!',
-        '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
-      ]
-
-      run_tests(d, tests)
-      compare_test_result(d)
-    end
-
-    def test_syslog_auto_format
-      d = create_driver(CONFIG + 'message_format auto')
-      tests = [
-        '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!',
-        '<6>Sep 11 00:00:00 localhost fluentd: Hi, from Fluentd!'
-      ]
-
-      run_tests(d, tests)
-      compare_test_result(d)
-    end
-
-    def run_tests(d, tests)
-      d.run do
-        u = Fluent::SocketUtil.create_udp_socket('127.0.0.1')
-        u.connect('127.0.0.1', PORT)
-        tests.each {|test|
-          u.send(test, 0)
-        }
-        sleep 1
-      end
-    end
-
-    def compare_test_result(d)
-      emits = d.emits
-      assert_equal 2, emits.size
-      emits.each_index {|i|
-        record =  emits[i][2]
-        assert_equal('fluentd', record['ident'])
-        assert_equal('Hi, from Fluentd!', record['message'])
-      }
-    end
-  end
 end