fluentd 0.12.40 → 1.6.2

data/test/plugin/test_in_tcp.rb

@@ -1,5 +1,6 @@
 require_relative '../helper'
-require 'fluent/test'
+require 'fluent/test/driver/input'
+require 'fluent/plugin/in_tcp'
 
 class TcpInputTest < Test::Unit::TestCase
   def setup
@@ -21,68 +22,144 @@ class TcpInputTest < Test::Unit::TestCase
   ]
 
   def create_driver(conf)
-    Fluent::Test::InputTestDriver.new(Fluent::TcpInput).configure(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::TcpInput).configure(conf)
   end
 
-  def test_configure
-    configs = {'127.0.0.1' => CONFIG}
-    configs.merge!('::1' => IPv6_CONFIG) if ipv6_enabled?
+  def create_tcp_socket(host, port, &block)
+    if block_given?
+      TCPSocket.open(host, port, &block)
+    else
+      TCPSocket.open(host, port)
+    end
+  end
+
+
+  data(
+    'ipv4' => [CONFIG, '127.0.0.1', :ipv4],
+    'ipv6' => [IPv6_CONFIG, '::1', :ipv6],
+  )
+  test 'configure' do |data|
+    conf, bind, protocol = data
+    omit "IPv6 is not supported on this environment" if protocol == :ipv6 && !ipv6_enabled?
+
+    d = create_driver(conf)
+    assert_equal PORT, d.instance.port
+    assert_equal bind, d.instance.bind
+    assert_equal "\n", d.instance.delimiter
+  end
 
-    configs.each_pair { |k, v|
-      d = create_driver(v)
-      assert_equal PORT, d.instance.port
-      assert_equal k, d.instance.bind
-      assert_equal "\n", d.instance.delimiter
+  test ' configure w/o parse section' do
+    assert_raise(Fluent::ConfigError.new("<parse> section is required.")) {
+      create_driver(BASE_CONFIG)
     }
   end
 
-  {
-    'none' => [
-      {'msg' => "tcptest1\n", 'expected' => 'tcptest1'},
-      {'msg' => "tcptest2\n", 'expected' => 'tcptest2'},
-    ],
-    'json' => [
-      {'msg' => {'k' => 123, 'message' => 'tcptest1'}.to_json + "\n", 'expected' => 'tcptest1'},
-      {'msg' => {'k' => 'tcptest2', 'message' => 456}.to_json + "\n", 'expected' => 456},
-    ]
-  }.each { |format, test_cases|
-    define_method("test_msg_size_#{format}") do
-      d = create_driver(BASE_CONFIG + "format #{format}")
-      tests = test_cases
-
-      d.run do
-        tests.each {|test|
-          TCPSocket.open('127.0.0.1', PORT) do |s|
-            s.send(test['msg'], 0)
-          end
-        }
-        sleep 1
+  test_case_data = {
+    'none' => {
+      'format' => 'none',
+      'payloads' => [ "tcptest1\n", "tcptest2\n" ],
+      'expecteds' => [
+        {'message' => 'tcptest1'},
+        {'message' => 'tcptest2'},
+      ],
+    },
+    'json' => {
+      'format' => 'json',
+      'payloads' => [
+        {'k' => 123, 'message' => 'tcptest1'}.to_json + "\n",
+        {'k' => 'tcptest2', 'message' => 456}.to_json + "\n",
+      ],
+      'expecteds' => [
+        {'k' => 123, 'message' => 'tcptest1'},
+        {'k' => 'tcptest2', 'message' => 456}
+      ],
+    },
+  }
+
+  data(test_case_data)
+  test 'test_msg_size' do |data|
+    format = data['format']
+    payloads = data['payloads']
+    expecteds = data['expecteds']
+
+    d = create_driver(BASE_CONFIG + "format #{format}")
+    d.run(expect_records: 2) do
+      payloads.each do |payload|
+        create_tcp_socket('127.0.0.1', PORT) do |sock|
+          sock.send(payload, 0)
+        end
       end
+    end
 
-      compare_test_result(d.emits, tests)
+    assert_equal 2, d.events.size
+    expecteds.each_with_index do |expected_record, i|
+      assert_equal "tcp", d.events[i][0]
+      assert d.events[i][1].is_a?(Fluent::EventTime)
+      assert_equal expected_record, d.events[i][2]
     end
+  end
 
-    define_method("test_msg_size_with_same_connection_#{format}") do
-      d = create_driver(BASE_CONFIG + "format #{format}")
-      tests = test_cases
+  data(test_case_data)
+  test 'test data in a connection' do |data|
+    format = data['format']
+    payloads = data['payloads']
+    expecteds = data['expecteds']
 
-      d.run do
-        TCPSocket.open('127.0.0.1', PORT) do |s|
-          tests.each {|test|
-            s.send(test['msg'], 0)
-          }
+    d = create_driver(BASE_CONFIG + "format #{format}")
+    d.run(expect_records: 2) do
+      create_tcp_socket('127.0.0.1', PORT) do |sock|
+        payloads.each do |payload|
+          sock.send(payload, 0)
         end
-        sleep 1
       end
+    end
 
-      compare_test_result(d.emits, tests)
+    assert_equal 2, d.events.size
+    expecteds.each_with_index do |expected_record, i|
+      assert_equal "tcp", d.events[i][0]
+      assert d.events[i][1].is_a?(Fluent::EventTime)
+      assert_equal expected_record, d.events[i][2]
     end
-  }
+  end
 
-  def compare_test_result(emits, tests)
-    assert_equal(2, emits.size)
-    emits.each_index {|i|
-      assert_equal(tests[i]['expected'], emits[i][2]['message'])
-    }
+  test 'source_hostname_key' do
+    d = create_driver(BASE_CONFIG + %!
+      format none
+      source_hostname_key host
+    !)
+    hostname = nil
+    d.run(expect_records: 1) do
+      create_tcp_socket('127.0.0.1', PORT) do |sock|
+        sock.do_not_reverse_lookup = false
+        hostname = sock.peeraddr[2]
+        sock.send("test\n", 0)
+      end
+    end
+
+    assert_equal 1, d.events.size
+    event = d.events[0]
+    assert_equal "tcp", event[0]
+    assert event[1].is_a?(Fluent::EventTime)
+    assert_equal hostname, event[2]['host']
+  end
+
+  test 'source_address_key' do
+    d = create_driver(BASE_CONFIG + %!
+      format none
+      source_address_key addr
+    !)
+    address = nil
+    d.run(expect_records: 1) do
+      create_tcp_socket('127.0.0.1', PORT) do |sock|
+        address = sock.peeraddr[3]
+        sock.send("test\n", 0)
+      end
+    end
+
+    assert_equal 1, d.events.size
+    event = d.events[0]
+    assert_equal "tcp", event[0]
+    assert event[1].is_a?(Fluent::EventTime)
+    assert_equal address, event[2]['addr']
   end
 end

data/test/plugin/test_in_udp.rb

@@ -1,5 +1,6 @@
 require_relative '../helper'
-require 'fluent/test'
+require 'fluent/test/driver/input'
+require 'fluent/plugin/in_udp'
 
 class UdpInputTest < Test::Unit::TestCase
   def setup
@@ -21,93 +22,233 @@ class UdpInputTest < Test::Unit::TestCase
   !
 
   def create_driver(conf)
-    Fluent::Test::InputTestDriver.new(Fluent::UdpInput).configure(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::UdpInput).configure(conf)
   end
 
-  def test_configure
-    configs = {'127.0.0.1' => CONFIG}
-    configs.merge!('::1' => IPv6_CONFIG) if ipv6_enabled?
+  def create_udp_socket(host, port)
+    u = if IPAddr.new(IPSocket.getaddress(host)).ipv4?
+          UDPSocket.new(Socket::AF_INET)
+        else
+          UDPSocket.new(Socket::AF_INET6)
+        end
+    u.do_not_reverse_lookup = false
+    u.connect(host, port)
+    if block_given?
+      begin
+        yield u
+      ensure
+        u.close rescue nil
+      end
+    else
+      u
+    end
+  end
+
+  data(
+    'ipv4' => [CONFIG, '127.0.0.1', :ipv4],
+    'ipv6' => [IPv6_CONFIG, '::1', :ipv6],
+  )
+  test 'configure' do |data|
+    conf, bind, protocol = data
+    omit "IPv6 is not supported on this environment" if protocol == :ipv6 && !ipv6_enabled?
 
-    configs.each_pair { |k, v|
-      d = create_driver(v)
-      assert_equal PORT, d.instance.port
-      assert_equal k, d.instance.bind
-      assert_equal 4096, d.instance.message_length_limit
+    d = create_driver(conf)
+    assert_equal PORT, d.instance.port
+    assert_equal bind, d.instance.bind
+    assert_equal 4096, d.instance.message_length_limit
+    assert_equal nil, d.instance.receive_buffer_size
+  end
+
+  test ' configure w/o parse section' do
+    assert_raise(Fluent::ConfigError.new("<parse> section is required.")) {
+      create_driver(BASE_CONFIG)
     }
   end
 
+  data(
+    'ipv4' => [CONFIG, '127.0.0.1', :ipv4],
+    'ipv6' => [IPv6_CONFIG, '::1', :ipv6],
+  )
+  test 'time_format' do |data|
+    conf, bind, protocol = data
+    omit "IPv6 is not supported on this environment" if protocol == :ipv6 && !ipv6_enabled?
+
+    d = create_driver(conf)
+
+    tests = [
+      {'msg' => '[Sep 11 00:00:00] localhost logger: foo', 'expected' => event_time('Sep 11 00:00:00', format: '%b %d %H:%M:%S')},
+      {'msg' => '[Sep  1 00:00:00] localhost logger: foo', 'expected' => event_time('Sep  1 00:00:00', format: '%b  %d %H:%M:%S')},
+    ]
+
+    d.run(expect_records: 2) do
+      create_udp_socket(bind, PORT) do |u|
+        tests.each do |test|
+          u.send(test['msg'], 0)
+        end
+      end
+    end
+
+    events = d.events
+    tests.each_with_index do |t, i|
+      assert_equal_event_time(t['expected'], events[i][1])
+    end
+  end
+
   data(
     'message_length_limit' => 'message_length_limit 2048',
     'body_size_limit' => 'body_size_limit 2048'
   )
   test 'message_length_limit/body_size_limit compatibility' do |param|
+
     d = create_driver(CONFIG + param)
     assert_equal 2048, d.instance.message_length_limit
   end
 
-  def test_time_format
-    configs = {'127.0.0.1' => CONFIG}
-    configs.merge!('::1' => IPv6_CONFIG) if ipv6_enabled?
+  data(
+    'none' => {
+      'format' => 'none',
+      'payloads' => ["tcptest1\n", "tcptest2\n"],
+      'expecteds' => [
+        {"message" => "tcptest1"},
+        {"message" => "tcptest2"},
+      ],
+    },
+    'json' => {
+      'format' => 'json',
+      'payloads' => [
+        {'k' => 123, 'message' => 'tcptest1'}.to_json + "\n",
+        {'k' => 'tcptest2', 'message' => 456}.to_json + "\n",
+      ],
+      'expecteds' => [
+        {'k' => 123, 'message' => 'tcptest1'},
+        {'k' => 'tcptest2', 'message' => 456},
+      ],
+    },
+    'regexp' => {
+      'format' => '/^\\[(?<time>[^\\]]*)\\] (?<message>.*)/',
+      'payloads' => [
+        '[Sep 10 00:00:00] localhost: ' + 'x' * 100 + "\n",
+        '[Sep 10 00:00:00] localhost: ' + 'x' * 1024 + "\n"
+      ],
+      'expecteds' => [
+        {"message" => 'localhost: ' + 'x' * 100},
+        {"message" => 'localhost: ' + 'x' * 1024},
+      ],
+    },
+  )
+  test 'message size with format' do |data|
+    format = data['format']
+    payloads = data['payloads']
+    expecteds = data['expecteds']
 
-    configs.each_pair { |k, v|
-      d = create_driver(v)
+    d = create_driver(BASE_CONFIG + "format #{format}")
+    d.run(expect_records: 2) do
+      create_udp_socket('127.0.0.1', PORT) do |u|
+        payloads.each do |payload|
+          u.send(payload, 0)
+        end
+      end
+    end
 
-      tests = [
-        {'msg' => '[Sep 11 00:00:00] localhost logger: foo', 'expected' => Time.strptime('Sep 11 00:00:00', '%b %d %H:%M:%S').to_i},
-        {'msg' => '[Sep  1 00:00:00] localhost logger: foo', 'expected' => Time.strptime('Sep  1 00:00:00', '%b  %d %H:%M:%S').to_i},
-      ]
+    assert_equal 2, d.events.size
+    expecteds.each_with_index do |expected_record, i|
+      assert_equal "udp", d.events[i][0]
+      assert d.events[i][1].is_a?(Fluent::EventTime)
+      assert_equal expected_record, d.events[i][2]
+    end
+  end
 
-      d.run do
-        u = Fluent::SocketUtil.create_udp_socket(k)
-        u.connect(k, PORT)
-        tests.each {|test|
-          u.send(test['msg'], 0)
-        }
-        sleep 1
+  test 'remove_newline' do
+    d = create_driver(BASE_CONFIG + %!
+      format none
+      remove_newline false
+    !)
+    payloads = ["test1\n", "test2\n"]
+    d.run(expect_records: 2) do
+      create_udp_socket('127.0.0.1', PORT) do |u|
+        payloads.each do |payload|
+          u.send(payload, 0)
+        end
       end
+    end
 
-      emits = d.emits
-      emits.each_index {|i|
-        assert_equal(tests[i]['expected'], emits[i][1])
-      }
-    }
+    expecteds = payloads.map { |payload| {'message' => payload} }
+    assert_equal 2, d.events.size
+    expecteds.each_with_index do |expected_record, i|
+      assert_equal "udp", d.events[i][0]
+      assert d.events[i][1].is_a?(Fluent::EventTime)
+      assert_equal expected_record, d.events[i][2]
+    end
   end
 
-  {
-    'none' => [
-      {'msg' => "tcptest1\n", 'expected' => 'tcptest1'},
-      {'msg' => "tcptest2\n", 'expected' => 'tcptest2'},
-    ],
-    'json' => [
-      {'msg' => {'k' => 123, 'message' => 'tcptest1'}.to_json + "\n", 'expected' => 'tcptest1'},
-      {'msg' => {'k' => 'tcptest2', 'message' => 456}.to_json + "\n", 'expected' => 456},
-    ],
-    '/^\\[(?<time>[^\\]]*)\\] (?<message>.*)/' => [
-      {'msg' => '[Sep 10 00:00:00] localhost: ' + 'x' * 100 + "\n", 'expected' => 'localhost: ' + 'x' * 100},
-      {'msg' => '[Sep 10 00:00:00] localhost: ' + 'x' * 1024 + "\n", 'expected' => 'localhost: ' + 'x' * 1024},
-    ]
-  }.each { |format, test_cases|
-    define_method("test_msg_size_#{format[0] == '/' ? 'regexp' : format}") do
-      d = create_driver(BASE_CONFIG + "format #{format}")
-      tests = test_cases
-
-      d.run do
-        u = UDPSocket.new
-        u.connect('127.0.0.1', PORT)
-        tests.each { |test|
-          u.send(test['msg'], 0)
-        }
-        sleep 1
+  test 'source_hostname_key' do
+    d = create_driver(BASE_CONFIG + %!
+      format none
+      source_hostname_key host
+    !)
+    hostname = nil
+    d.run(expect_records: 1) do
+      create_udp_socket('127.0.0.1', PORT) do |u|
+        u.send("test", 0)
+        hostname = u.peeraddr[2]
       end
+    end
 
-      compare_test_result(d.emits, tests)
+    assert_equal 1, d.events.size
+    assert_equal "udp", d.events[0][0]
+    assert d.events[0][1].is_a?(Fluent::EventTime)
+    assert_equal hostname, d.events[0][2]['host']
+  end
+
+  test 'source_address_key' do
+    d = create_driver(BASE_CONFIG + %!
+      format none
+      source_address_key addr
+    !)
+    address = nil
+    d.run(expect_records: 1) do
+      create_udp_socket('127.0.0.1', PORT) do |u|
+        u.send("test", 0)
+        address = u.peeraddr[3]
+      end
     end
-  }
 
-  def compare_test_result(emits, tests)
-    assert_equal(2, emits.size)
-    emits.each_index {|i|
-      assert_equal(tests[i]['expected'], emits[i][2]['message'])
-    }
+    assert_equal 1, d.events.size
+    assert_equal "udp", d.events[0][0]
+    assert d.events[0][1].is_a?(Fluent::EventTime)
+    assert_equal address, d.events[0][2]['addr']
+  end
+
+  test 'receive_buffer_size' do
+    # doesn't check exact value because it depends on platform and condition
+
+    # check if default socket and in_udp's one without receive_buffer_size have same size buffer
+    d0 = create_driver(BASE_CONFIG + %!
+      format none
+    !)
+    d0.run do
+      sock = d0.instance.instance_variable_get(:@_servers)[0].server.instance_variable_get(:@sock)
+      begin
+        default_sock = UDPSocket.new
+        assert_equal(default_sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int, sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int)
+      ensure
+        default_sock.close
+      end
+    end
+
+    # check if default socket and in_udp's one with receive_buffer_size have different size buffer
+    d1 = create_driver(BASE_CONFIG + %!
+      format none
+      receive_buffer_size 1001
+    !)
+    d1.run do
+      sock = d1.instance.instance_variable_get(:@_servers)[0].server.instance_variable_get(:@sock)
+      begin
+        default_sock = UDPSocket.new
+        assert_not_equal(default_sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int, sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int)
+      ensure
+        default_sock.close
+      end
+    end
   end
 end

data/test/plugin/{test_in_stream.rb → test_in_unix.rb}

@@ -1,5 +1,6 @@
 require_relative '../helper'
 require 'fluent/test'
+require 'fluent/plugin/in_unix'
 
 module StreamInputTest
   def setup
@@ -9,14 +10,14 @@ module StreamInputTest
   def test_time
     d = create_driver
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
     Fluent::Engine.now = time
 
     d.expect_emit "tag1", time, {"a"=>1}
     d.expect_emit "tag2", time, {"a"=>2}
 
     d.run do
-      d.expected_emits.each {|tag,time,record|
+      d.expected_emits.each {|tag,_time,record|
         send_data Fluent::Engine.msgpack_factory.packer.write([tag, 0, record]).to_s
       }
     end
@@ -25,14 +26,14 @@ module StreamInputTest
   def test_message
     d = create_driver
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
 
     d.expect_emit "tag1", time, {"a"=>1}
     d.expect_emit "tag2", time, {"a"=>2}
 
     d.run do
-      d.expected_emits.each {|tag,time,record|
-        send_data Fluent::Engine.msgpack_factory.packer.write([tag, time, record]).to_s
+      d.expected_emits.each {|tag,_time,record|
+        send_data Fluent::Engine.msgpack_factory.packer.write([tag, _time, record]).to_s
       }
     end
   end
@@ -40,15 +41,15 @@ module StreamInputTest
   def test_forward
     d = create_driver
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
 
     d.expect_emit "tag1", time, {"a"=>1}
     d.expect_emit "tag1", time, {"a"=>2}
 
     d.run do
       entries = []
-      d.expected_emits.each {|tag,time,record|
-        entries << [time, record]
+      d.expected_emits.each {|tag,_time,record|
+        entries << [_time, record]
       }
       send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
     end
@@ -57,15 +58,15 @@ module StreamInputTest
   def test_packed_forward
     d = create_driver
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
 
     d.expect_emit "tag1", time, {"a"=>1}
     d.expect_emit "tag1", time, {"a"=>2}
 
     d.run do
       entries = ''
-      d.expected_emits.each {|tag,time,record|
-        Fluent::Engine.msgpack_factory.packer(entries).write([time, record]).flush
+      d.expected_emits.each {|tag,_time,record|
+        Fluent::Engine.msgpack_factory.packer(entries).write([_time, record]).flush
       }
       send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
     end
@@ -80,7 +81,7 @@ module StreamInputTest
     d.expect_emit "tag2", time, {"a"=>2}
 
     d.run do
-      d.expected_emits.each {|tag,time,record|
+      d.expected_emits.each {|tag,_time,record|
         send_data [tag, time, record].to_json
       }
     end
@@ -122,4 +123,4 @@ class UnixInputTest < Test::Unit::TestCase
   def connect
     UNIXSocket.new("#{TMP_DIR}/unix")
   end
-end
+end unless Fluent.windows?