fluentd 0.14.17-x86-mingw32 → 1.3.1-x86-mingw32

data/test/plugin/test_in_monitor_agent.rb

@@ -455,7 +455,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
       d.instance.start
       expected_test_out_fail_write_response = {
           "buffer_queue_length" => 1,
-          "buffer_total_queued_size" => 0,
+          "buffer_total_queued_size" => 40,
           "output_plugin" => true,
           "plugin_category" => "output",
           "plugin_id" => "test_out_fail_write",

data/test/plugin/test_in_syslog.rb

@@ -37,6 +37,24 @@ class SyslogInputTest < Test::Unit::TestCase
     assert_equal bind_addr, d.instance.bind
   end
 
+  sub_test_case 'source_hostname_key and source_address_key features' do
+    test 'resolve_hostname must be true with source_hostname_key' do
+      assert_raise(Fluent::ConfigError) {
+        create_driver(CONFIG + <<EOS)
+resolve_hostname false
+source_hostname_key hostname
+EOS
+      }
+    end
+
+    data('resolve_hostname' => 'resolve_hostname true',
+         'source_hostname_key' => 'source_hostname_key source_host')
+    def test_configure_resolve_hostname(param)
+      d = create_driver([CONFIG, param].join("\n"))
+      assert_true d.instance.resolve_hostname
+    end
+  end
+
   data(
     ipv4: ['127.0.0.1', CONFIG, ::Socket::AF_INET],
     ipv6: ['::1', IPv6_CONFIG, ::Socket::AF_INET6],
@@ -268,4 +286,50 @@ class SyslogInputTest < Test::Unit::TestCase
       assert_equal(options[:facility], events[i][2]['facility']) if options[:facility]
     }
   end
+
+  sub_test_case 'octet counting frame' do
+    def test_msg_size_with_tcp
+      d = create_driver([CONFIG, 'protocol_type tcp', 'frame_type octet_count'].join("\n"))
+      tests = create_test_case
+
+      d.run(expect_emits: 2) do
+        tests.each {|test|
+          TCPSocket.open('127.0.0.1', PORT) do |s|
+            s.send(test['msg'], 0)
+          end
+        }
+      end
+
+      assert(d.events.size > 0)
+      compare_test_result(d.events, tests)
+    end
+
+    def test_msg_size_with_same_tcp_connection
+      d = create_driver([CONFIG, 'protocol_type tcp', 'frame_type octet_count'].join("\n"))
+      tests = create_test_case
+
+      d.run(expect_emits: 2) do
+        TCPSocket.open('127.0.0.1', PORT) do |s|
+          tests.each {|test|
+            s.send(test['msg'], 0)
+          }
+        end
+      end
+
+      assert(d.events.size > 0)
+      compare_test_result(d.events, tests)
+    end
+
+    def create_test_case(large_message: false)
+      msgs = [
+        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100, 'expected' => 'x' * 100},
+        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 1024, 'expected' => 'x' * 1024},
+      ]
+      msgs.each { |msg|
+        m = msg['msg']
+        msg['msg'] = "#{m.size + 1} #{m}"
+      }
+      msgs
+    end
+  end
 end

data/test/plugin/test_in_tail.rb

@@ -41,6 +41,7 @@ class TailInputTest < Test::Unit::TestCase
   COMMON_CONFIG = CONFIG + config_element("", "", { "pos_file" => "#{TMP_DIR}/tail.pos" })
   CONFIG_READ_FROM_HEAD = config_element("", "", { "read_from_head" => true })
   CONFIG_ENABLE_WATCH_TIMER = config_element("", "", { "enable_watch_timer" => false })
+  CONFIG_DISABLE_STAT_WATCHER = config_element("", "", { "enable_stat_watcher" => false })
   CONFIG_OPEN_ON_EVERY_UPDATE = config_element("", "", { "open_on_every_update" => true })
   SINGLE_LINE_CONFIG = config_element("", "", { "format" => "/(?<message>.*)/" })
   PARSE_SINGLE_LINE_CONFIG = config_element("", "", {}, [config_element("parse", "", { "@type" => "/(?<message>.*)/" })])
@@ -83,6 +84,12 @@ class TailInputTest < Test::Unit::TestCase
       end
     end
 
+    test "both enable_watch_timer and enable_stat_watcher are false" do
+      assert_raise(Fluent::ConfigError) do
+        create_driver(CONFIG_ENABLE_WATCH_TIMER + CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
+      end
+    end
+
     sub_test_case "encoding" do
       test "valid" do
         conf = SINGLE_LINE_CONFIG + config_element("", "", { "encoding" => "utf-8" })
@@ -256,6 +263,30 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal({"message" => "test3"}, events[0][2])
       assert_equal({"message" => "test4"}, events[1][2])
     end
+
+    data(flat: CONFIG_DISABLE_STAT_WATCHER + SINGLE_LINE_CONFIG,
+         parse: CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
+    def test_emit_with_disable_stat_watcher(data)
+      config = data
+      File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+        f.puts "test1"
+        f.puts "test2"
+      }
+
+      d = create_driver(config)
+
+      d.run(expect_emits: 1) do
+        File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
+          f.puts "test3"
+          f.puts "test4"
+        }
+      end
+
+      events = d.events
+      assert(events.length > 0)
+      assert_equal({"message" => "test3"}, events[0][2])
+      assert_equal({"message" => "test4"}, events[1][2])
+    end
   end
 
   class TestWithSystem < self
@@ -440,6 +471,71 @@ class TailInputTest < Test::Unit::TestCase
     end
   end
 
+  def test_truncate_file
+    omit "Permission denied error happen on Windows. Need fix" if Fluent.windows?
+
+    config = SINGLE_LINE_CONFIG
+    File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+      f.puts "test1"
+      f.puts "test2"
+    }
+
+    d = create_driver(config)
+
+    d.run(expect_emits: 2) do
+      File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
+        f.puts "test3\ntest4"
+        f.flush
+      }
+      sleep 1
+      File.truncate("#{TMP_DIR}/tail.txt", 6)
+    end
+
+    events = d.events
+    assert_equal(3, events.length)
+    assert_equal({"message" => "test3"}, events[0][2])
+    assert_equal({"message" => "test4"}, events[1][2])
+    assert_equal({"message" => "test1"}, events[2][2])
+    assert(events[0][1].is_a?(Fluent::EventTime))
+    assert(events[1][1].is_a?(Fluent::EventTime))
+    assert(events[2][1].is_a?(Fluent::EventTime))
+    assert_equal(2, d.emit_count)
+  end
+
+  def test_move_truncate_move_back
+    omit "Permission denied error happen on Windows. Need fix" if Fluent.windows?
+
+    config = SINGLE_LINE_CONFIG
+    File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
+      f.puts "test1"
+      f.puts "test2"
+    }
+
+    d = create_driver(config)
+
+    d.run(expect_emits: 1) do
+      if Fluent.windows?
+        FileUtils.mv("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail2.txt", force: true)
+      else
+        FileUtils.mv("#{TMP_DIR}/tail.txt", "#{TMP_DIR}/tail2.txt")
+      end
+      sleep(1)
+      File.truncate("#{TMP_DIR}/tail2.txt", 6)
+      sleep(1)
+      if Fluent.windows?
+        FileUtils.mv("#{TMP_DIR}/tail2.txt", "#{TMP_DIR}/tail.txt", force: true)
+      else
+        FileUtils.mv("#{TMP_DIR}/tail2.txt", "#{TMP_DIR}/tail.txt")
+      end
+    end
+
+    events = d.events
+    assert_equal(1, events.length)
+    assert_equal({"message" => "test1"}, events[0][2])
+    assert(events[0][1].is_a?(Fluent::EventTime))
+    assert_equal(1, d.emit_count)
+  end
+
   def test_lf
     File.open("#{TMP_DIR}/tail.txt", "wb") {|f| }
 
@@ -823,7 +919,7 @@ class TailInputTest < Test::Unit::TestCase
   sub_test_case "path" do
     # * path test
     # TODO: Clean up tests
-    EX_RORATE_WAIT = 0
+    EX_ROTATE_WAIT = 0
 
     EX_CONFIG = config_element("", "", {
                                  "tag" => "tail",
@@ -832,7 +928,7 @@ class TailInputTest < Test::Unit::TestCase
                                  "pos_file" => "#{TMP_DIR}/tail.pos",
                                  "read_from_head" => true,
                                  "refresh_interval" => 30,
-                                 "rotate_wait" => "#{EX_RORATE_WAIT}s",
+                                 "rotate_wait" => "#{EX_ROTATE_WAIT}s",
                                })
     EX_PATHS = [
       'test/plugin/data/2010/01/20100102-030405.log',
@@ -873,7 +969,7 @@ class TailInputTest < Test::Unit::TestCase
     end
 
     # For https://github.com/fluent/fluentd/issues/1455
-    # This test is fragile because test content depends on internal implementaion.
+    # This test is fragile because test content depends on internal implementation.
     # So if you modify in_tail internal, this test may break.
     def test_unwatched_files_should_be_removed
       config = config_element("", "", {
@@ -894,7 +990,7 @@ class TailInputTest < Test::Unit::TestCase
       waiting(20) { sleep 0.1 until Dir.glob("#{TMP_DIR}/*.txt").size == 0 } # Ensure file is deleted on Windows
       waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size == 0 }
 
-      # Previous implementaion has an infinite watcher creation bug.
+      # Previous implementation has an infinite watcher creation bug.
       # Following code checks such unexpected bug by couting  actual object allocation.
       base_num = count_timer_object
       2.times {
@@ -926,7 +1022,7 @@ class TailInputTest < Test::Unit::TestCase
     Timecop.freeze(2010, 1, 2, 3, 4, 5) do
       flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
         EX_PATHS.each do |path|
-          watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, 1000, any, any, any, any, any, any).once.and_return do
+          watcherclass.should_receive(:new).with(path, EX_ROTATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, true, 1000, any, any, any, any, any, any).once.and_return do
             flexmock('TailWatcher') { |watcher|
               watcher.should_receive(:attach).once
               watcher.should_receive(:unwatched=).zero_or_more_times
@@ -944,7 +1040,7 @@ class TailInputTest < Test::Unit::TestCase
 
     Timecop.freeze(2010, 1, 2, 3, 4, 6) do
       flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
-        watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, 1000, any, any, any, any, any, any).once.and_return do
+        watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_ROTATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, true, 1000, any, any, any, any, any, any).once.and_return do
           flexmock('TailWatcher') do |watcher|
             watcher.should_receive(:attach).once
             watcher.should_receive(:unwatched=).zero_or_more_times
@@ -972,6 +1068,20 @@ class TailInputTest < Test::Unit::TestCase
       plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
     end
 
+    def test_tag_with_only_star
+      config = config_element("", "", {
+                                "tag" => "*",
+                                "path" => "test/plugin/*/%Y/%m/%Y%m%d-%H%M%S.log,test/plugin/data/log/**/*.log",
+                                "format" => "none",
+                                "read_from_head" => true
+                              })
+      d = create_driver(config, false)
+      d.run {}
+      plugin = d.instance
+      mock(plugin.router).emit_stream('foo.bar.log', anything).once
+      plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
+    end
+
     def test_tag_prefix
       config = config_element("", "", {
                                 "tag" => "pre.*",

data/test/plugin/test_in_tcp.rb

@@ -115,4 +115,25 @@ class TcpInputTest < Test::Unit::TestCase
       assert_equal expected_record, d.events[i][2]
     end
   end
+
+  test 'source_hostname_key' do
+    d = create_driver(BASE_CONFIG + %!
+      format none
+      source_hostname_key host
+    !)
+    hostname = nil
+    d.run(expect_records: 1) do
+      create_tcp_socket('127.0.0.1', PORT) do |sock|
+        sock.do_not_reverse_lookup = false
+        hostname = sock.peeraddr[2]
+        sock.send("test\n", 0)
+      end
+    end
+
+    assert_equal 1, d.events.size
+    event = d.events[0]
+    assert_equal "tcp", event[0]
+    assert event[1].is_a?(Fluent::EventTime)
+    assert_equal hostname, event[2]['host']
+  end
 end

data/test/plugin/test_in_udp.rb

@@ -31,6 +31,7 @@ class UdpInputTest < Test::Unit::TestCase
         else
           UDPSocket.new(Socket::AF_INET6)
         end
+    u.do_not_reverse_lookup = false
     u.connect(host, port)
     if block_given?
       begin
@@ -55,6 +56,7 @@ class UdpInputTest < Test::Unit::TestCase
     assert_equal PORT, d.instance.port
     assert_equal bind, d.instance.bind
     assert_equal 4096, d.instance.message_length_limit
+    assert_equal nil, d.instance.receive_buffer_size
   end
 
   data(
@@ -149,4 +151,80 @@ class UdpInputTest < Test::Unit::TestCase
       assert_equal expected_record, d.events[i][2]
     end
   end
+
+  test 'remove_newline' do
+    d = create_driver(BASE_CONFIG + %!
+      format none
+      remove_newline false
+    !)
+    payloads = ["test1\n", "test2\n"]
+    d.run(expect_records: 2) do
+      create_udp_socket('127.0.0.1', PORT) do |u|
+        payloads.each do |payload|
+          u.send(payload, 0)
+        end
+      end
+    end
+
+    expecteds = payloads.map { |payload| {'message' => payload} }
+    assert_equal 2, d.events.size
+    expecteds.each_with_index do |expected_record, i|
+      assert_equal "udp", d.events[i][0]
+      assert d.events[i][1].is_a?(Fluent::EventTime)
+      assert_equal expected_record, d.events[i][2]
+    end
+  end
+
+  test 'source_hostname_key' do
+    d = create_driver(BASE_CONFIG + %!
+      format none
+      source_hostname_key host
+    !)
+    hostname = nil
+    d.run(expect_records: 1) do
+      create_udp_socket('127.0.0.1', PORT) do |u|
+        u.send("test", 0)
+        hostname = u.peeraddr[2]
+      end
+    end
+
+    expected = {'message' => 'test'}
+    assert_equal 1, d.events.size
+    assert_equal "udp", d.events[0][0]
+    assert d.events[0][1].is_a?(Fluent::EventTime)
+    assert_equal hostname, d.events[0][2]['host']
+  end
+
+  test 'receive_buffer_size' do
+    # doesn't check exact value because it depends on platform and condition
+
+    # check if default socket and in_udp's one without receive_buffer_size have same size buffer
+    d0 = create_driver(BASE_CONFIG + %!
+      format none
+    !)
+    d0.run do
+      sock = d0.instance.instance_variable_get(:@_servers)[0].server.instance_variable_get(:@sock)
+      begin
+        default_sock = UDPSocket.new
+        assert_equal(default_sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int, sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int)
+      ensure
+        default_sock.close
+      end
+    end
+
+    # check if default socket and in_udp's one with receive_buffer_size have different size buffer
+    d1 = create_driver(BASE_CONFIG + %!
+      format none
+      receive_buffer_size 1001
+    !)
+    d1.run do
+      sock = d1.instance.instance_variable_get(:@_servers)[0].server.instance_variable_get(:@sock)
+      begin
+        default_sock = UDPSocket.new
+        assert_not_equal(default_sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int, sock.getsockopt(Socket::SOL_SOCKET, Socket::SO_RCVBUF).int)
+      ensure
+        default_sock.close
+      end
+    end
+  end
 end

data/test/plugin/test_metadata.rb

@@ -0,0 +1,89 @@
+require_relative '../helper'
+require 'fluent/plugin/buffer'
+
+class BufferMetadataTest < Test::Unit::TestCase
+
+  def meta(timekey=nil, tag=nil, variables=nil)
+    Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
+  end
+
+  setup do
+    Fluent::Test.setup
+  end
+
+  sub_test_case 'about metadata' do
+    test 'comparison of variables should be stable' do
+      m = meta(nil, nil, nil)
+      # different sets of keys
+      assert_equal(-1, m.cmp_variables({}, {a: 1}))
+      assert_equal(1, m.cmp_variables({a: 1}, {}))
+      assert_equal(1, m.cmp_variables({c: 1}, {a: 1}))
+      assert_equal(-1, m.cmp_variables({a: 1}, {a: 1, b: 2}))
+      assert_equal(1, m.cmp_variables({a: 1, c: 1}, {a: 1, b: 2}))
+      assert_equal(1, m.cmp_variables({a: 1, b: 0, c: 1}, {a: 1, b: 2}))
+      # same set of keys
+      assert_equal(-1, m.cmp_variables({a: 1}, {a: 2}))
+      assert_equal(-1, m.cmp_variables({a: 1, b: 0}, {a: 1, b: 1}))
+      assert_equal(-1, m.cmp_variables({a: 1, b: 1, c: 100}, {a: 1, b: 1, c: 200}))
+      assert_equal(-1, m.cmp_variables({b: 1, c: 100, a: 1}, {a: 1, b: 1, c: 200})) # comparison sorts keys
+      assert_equal(-1, m.cmp_variables({a: nil}, {a: 1}))
+      assert_equal(-1, m.cmp_variables({a: 1, b: nil}, {a: 1, b: 1}))
+    end
+
+    test 'comparison of metadata should be stable' do
+      n = Time.now.to_i
+
+      assert_equal(0, meta(nil, nil, nil) <=> meta(nil, nil, nil))
+      assert_equal(0, meta(n, nil, nil) <=> meta(n, nil, nil))
+      assert_equal(0, meta(nil, "t1", nil) <=> meta(nil, "t1", nil))
+      assert_equal(0, meta(nil, nil, {}) <=> meta(nil, nil, {}))
+      assert_equal(0, meta(nil, nil, {a: "1"}) <=> meta(nil, nil, {a: "1"}))
+      assert_equal(0, meta(n, nil, {}) <=> meta(n, nil, {}))
+      assert_equal(0, meta(n, "t1", {}) <=> meta(n, "t1", {}))
+      assert_equal(0, meta(n, "t1", {a: "x", b: 10}) <=> meta(n, "t1", {a: "x", b: 10}))
+
+      # timekey is 1st comparison key
+      assert_equal(-1, meta(n - 300, nil, nil) <=> meta(n - 270, nil, nil))
+      assert_equal(1, meta(n + 1, "a", nil) <=> meta(n - 1, "b", nil))
+      assert_equal(-1, meta(n - 1, nil, {a: 100}) <=> meta(n + 1, nil, {}))
+
+      # tag is 2nd
+      assert_equal(-1, meta(nil, "a", {}) <=> meta(nil, "b", {}))
+      assert_equal(-1, meta(n, "a", {}) <=> meta(n, "b", {}))
+      assert_equal(1, meta(nil, "x", {a: 1}) <=> meta(nil, "t", {}))
+      assert_equal(1, meta(n, "x", {a: 1}) <=> meta(n, "t", {}))
+      assert_equal(1, meta(nil, "x", {a: 1}) <=> meta(nil, "t", {a: 1}))
+      assert_equal(1, meta(n, "x", {a: 1}) <=> meta(n, "t", {a: 2}))
+      assert_equal(1, meta(n, "x", {a: 1}) <=> meta(n, "t", {a: 10, b: 1}))
+
+      # variables is the last
+      assert_equal(-1, meta(nil, nil, {}) <=> meta(nil, nil, {a: 1}))
+      assert_equal(-1, meta(n, "t", {}) <=> meta(n, "t", {a: 1}))
+      assert_equal(1, meta(n, "t", {a: 1}) <=> meta(n, "t", {}))
+      assert_equal(-1, meta(n, "t", {a: 1}) <=> meta(n, "t", {a: 2}))
+      assert_equal(-1, meta(n, "t", {a: 1}) <=> meta(n, "t", {a: 1, b: 1}))
+      assert_equal(1, meta(nil, nil, {b: 1}) <=> meta(nil, nil, {a: 1}))
+      assert_equal(1, meta(n, "t", {b: 1}) <=> meta(n, "t", {a: 1}))
+    end
+
+    test 'metadata can be sorted' do
+      n = Time.now.to_i
+      m0 = meta(nil, nil, nil)
+      m1 = meta(n - 1, nil, nil)
+      m2 = meta(n - 1, "a", nil)
+      m3 = meta(n - 1, "a", {a: 1})
+      m4 = meta(n - 1, "a", {a: 100})
+      m5 = meta(n - 1, "a", {a: 100, b: 1})
+      m6 = meta(n - 1, "aa", nil)
+      m7 = meta(n - 1, "aa", {a: 1})
+      m8 = meta(n - 1, "b", nil)
+      m9 = meta(n, nil, nil)
+      m10 = meta(n + 1, nil, {a: 1})
+      expected = [m0, m1, m2, m3, m4, m5, m6, m7, m8, m9, m10].freeze
+      ary = expected.dup
+      100.times do
+        assert_equal expected, ary.shuffle.sort
+      end
+    end
+  end
+end