fluentd 1.10.0-x64-mingw32 → 1.11.0-x64-mingw32

data/test/plugin/out_forward/test_load_balancer.rb

@@ -49,6 +49,44 @@ class LoadBalancerTest < Test::Unit::TestCase
       end
     end
 
+    test 'call like round robin without weight=0 node' do
+      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
+      n1 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
+      n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
+      n3 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 0)
+
+      lb.rebuild_weight_array([n1, n2, n3])
+
+      lb.select_healthy_node do |node|
+        # to handle random choice
+        if node == n1
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+        else
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+        end
+      end
+    end
+
     test 'raise an error if all node are unavialble' do
       lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
       lb.rebuild_weight_array([flexmock('node', :'standby?' => false, :'available?' => false, weight: 1)])
@@ -56,5 +94,13 @@ class LoadBalancerTest < Test::Unit::TestCase
         lb.select_healthy_node
       end
     end
+
+    test 'it regards weight=0 node as unavialble' do
+      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
+      lb.rebuild_weight_array([flexmock('node', :'standby?' => false, :'available?' => true, weight: 0)])
+      assert_raise(Fluent::Plugin::ForwardOutput::NoNodesAvailable) do
+        lb.select_healthy_node
+      end
+    end
   end
 end

data/test/plugin/test_in_gc_stat.rb

@@ -22,9 +22,14 @@ class GCStatInputTest < Test::Unit::TestCase
     assert_equal("t1", d.instance.tag)
   end
 
-  def test_emit
+  def setup_gc_stat
     stat = GC.stat
     stub(GC).stat { stat }
+    stat
+  end
+
+  def test_emit
+    stat = setup_gc_stat
 
     d = create_driver
     d.run(expect_emits: 2)
@@ -36,4 +41,22 @@ class GCStatInputTest < Test::Unit::TestCase
       assert(events[i][1].is_a?(Fluent::EventTime))
     }
   end
+
+  def test_emit_with_use_symbol_keys_false
+    stat = setup_gc_stat
+    result = {}
+    stat.each_pair { |k, v|
+      result[k.to_s] = v
+    }
+
+    d = create_driver(CONFIG + "use_symbol_keys false")
+    d.run(expect_emits: 2)
+
+    events = d.events
+    assert(events.length > 0)
+    events.each_index {|i|
+      assert_equal(result, events[i][2])
+      assert(events[i][1].is_a?(Fluent::EventTime))
+    }
+  end
 end

data/test/plugin/test_in_syslog.rb

@@ -163,6 +163,21 @@ EOS
     compare_test_result(d.events, tests)
   end
 
+  def test_emit_rfc5452
+    d = create_driver([CONFIG, "facility_key pri\n<parse>\n message_format rfc5424\nwith_priority true\n</parse>"].join("\n"))
+    msg = '<1>1 2017-02-06T13:14:15.003Z myhostname 02abaf0687f5 10339 02abaf0687f5 - method=POST db=0.00'
+
+    d.run(expect_emits: 1, timeout: 2) do
+      u = UDPSocket.new
+      u.connect('127.0.0.1', PORT)
+      u.send(msg, 0)
+    end
+
+    tag, _, event = d.events[0]
+    assert_equal('syslog.kern.alert', tag)
+    assert_equal('kern', event['pri'])
+  end
+
   def test_msg_size_with_same_tcp_connection
     d = create_driver([CONFIG, "<transport tcp> \n</transport>"].join("\n"))
     tests = create_test_case
@@ -360,7 +375,7 @@ EOS
       ]
       msgs.each { |msg|
         m = msg['msg']
-        msg['msg'] = "#{m.size + 1} #{m}"
+        msg['msg'] = "#{m.size} #{m}"
       }
       msgs
     end

data/test/plugin/test_in_tail.rb

@@ -23,9 +23,6 @@ class TailInputTest < Test::Unit::TestCase
   def cleanup_directory(path)
     FileUtils.rm_rf(path, secure: true)
     if File.exist?(path)
-      # ensure files are closed for Windows, on which deleted files
-      # are still visible from filesystem
-      GC.start(full_mark: true, immediate_mark: true, immediate_sweep: true)
       FileUtils.remove_entry_secure(path, true)
     end
     FileUtils.mkdir_p(path)
@@ -105,6 +102,12 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal ["tail.txt", "test2", "tmp,dev"], d.instance.paths
     end
 
+    test "multi paths with same path configured twice" do
+      c = config_element("ROOT", "", { "path" => "test1.txt,test2.txt,test1.txt", "tag" => "t1", "path_delimiter" => "," })
+      d = create_driver(c + PARSE_SINGLE_LINE_CONFIG, false)
+      assert_equal ["test2.txt","test1.txt"].sort, d.instance.paths.sort
+    end
+
     test "multi paths with invaid path_delimiter" do
       c = config_element("ROOT", "", { "path" => "tail.txt|test2|tmp,dev", "tag" => "t1", "path_delimiter" => "*" })
       assert_raise(Fluent::ConfigError) do
@@ -1025,6 +1028,17 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal EX_PATHS - [EX_PATHS.last], plugin.expand_paths.sort
     end
 
+    def test_expand_paths_with_duplicate_configuration
+      expanded_paths = [
+        'test/plugin/data/log/foo/bar.log',
+        'test/plugin/data/log/test.log'
+      ]
+      duplicate_config = EX_CONFIG.dup
+      duplicate_config["path"]="test/plugin/data/log/**/*.log, test/plugin/data/log/**/*.log"
+      plugin = create_driver(EX_CONFIG, false).instance
+      assert_equal expanded_paths, plugin.expand_paths.sort
+    end
+
     def test_expand_paths_with_timezone
       ['Asia/Taipei', '+08'].each do |tz_type|
         taipei_config = EX_CONFIG + config_element("", "", {"path_timezone" => tz_type})
@@ -1065,9 +1079,6 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal expected_files, plugin.expand_paths.sort
     end
 
-    # For https://github.com/fluent/fluentd/issues/1455
-    # This test is fragile because test content depends on internal implementation.
-    # So if you modify in_tail internal, this test may break.
     def test_unwatched_files_should_be_removed
       config = config_element("", "", {
                                 "tag" => "tail",
@@ -1079,22 +1090,15 @@ class TailInputTest < Test::Unit::TestCase
                               })
       d = create_driver(config, false)
       d.end_if { d.instance.instance_variable_get(:@tails).keys.size >= 1 }
-      d.run(expect_emits: 1, shutdown: false, timeout: 1) do
+      d.run(expect_emits: 1, shutdown: false) do
         File.open("#{TMP_DIR}/tail.txt", "ab") { |f| f.puts "test3\n" }
       end
 
       cleanup_directory(TMP_DIR)
       waiting(20) { sleep 0.1 until Dir.glob("#{TMP_DIR}/*.txt").size == 0 } # Ensure file is deleted on Windows
-      waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size == 0 }
+      waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size <= 0 }
 
-      # Previous implementation has an infinite watcher creation bug.
-      # Following code checks such unexpected bug by counting actual object allocation.
-      base_num = count_timer_object
-      2.times {
-        sleep 1
-        num = count_timer_object
-        assert_equal base_num, num
-      }
+      assert_equal 0, d.instance.instance_variable_get(:@tails).keys.size
 
       d.instance_shutdown
     end
@@ -1183,6 +1187,25 @@ class TailInputTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case "refresh of pos file" do
+    test 'type of pos_file_compaction_interval is time' do
+      tail = {
+        "tag" => "tail",
+        "path" => "#{TMP_DIR}/*.txt",
+        "format" => "none",
+        "pos_file" => "#{TMP_DIR}/pos/tail.pos",
+        "refresh_interval" => 1,
+        "read_from_head" => true,
+        'pos_file_compaction_interval' => '24h',
+      }
+      config = config_element("", "", tail)
+      d = create_driver(config, false)
+      mock(d.instance).timer_execute(:in_tail_refresh_watchers, 1.0).once
+      mock(d.instance).timer_execute(:in_tail_refresh_compact_pos_file, 60 * 60 * 24).once
+      d.run                     # call start
+    end
+  end
+
   sub_test_case "receive_lines" do
     DummyWatcher = Struct.new("DummyWatcher", :tag)
 

data/test/plugin/test_in_unix.rb

@@ -1,126 +1,181 @@
 require_relative '../helper'
-require 'fluent/test'
+require 'fluent/test/driver/input'
 require 'fluent/plugin/in_unix'
 
-module StreamInputTest
+class UnixInputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
+    @d = nil
   end
 
-  def test_time
-    d = create_driver
+  def teardown
+    @d.instance_shutdown if @d
+  end
 
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
-    Fluent::Engine.now = time
+  TMP_DIR = File.dirname(__FILE__) + "/../tmp/in_unix#{ENV['TEST_ENV_NUMBER']}"
+  CONFIG = %[
+    path #{TMP_DIR}/unix
+    backlog 1000
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::UnixInput).configure(conf)
+  end
 
-    d.expect_emit "tag1", time, {"a"=>1}
-    d.expect_emit "tag2", time, {"a"=>2}
+  def packer(*args)
+    Fluent::MessagePackFactory.msgpack_packer(*args)
+  end
 
-    d.run do
-      d.expected_emits.each {|tag,_time,record|
-        send_data Fluent::MessagePackFactory.msgpack_packer.write([tag, 0, record]).to_s
+  def unpacker
+    Fluent::MessagePackFactory.msgpack_unpacker
+  end
+
+  def send_data(data)
+    io = UNIXSocket.new("#{TMP_DIR}/unix")
+    begin
+      io.write data
+    ensure
+      io.close
+    end
+  end
+
+  def test_configure
+    @d = create_driver
+    assert_equal "#{TMP_DIR}/unix", @d.instance.path
+    assert_equal 1000, @d.instance.backlog
+  end
+
+  def test_time
+    @d = create_driver
+
+    time = Fluent::EventTime.now
+    records = [
+      ["tag1", 0, {"a" => 1}],
+      ["tag2", nil, {"a" => 2}],
+    ]
+
+    @d.run(expect_records: records.length, timeout: 5) do
+      records.each {|tag, _time, record|
+        send_data packer.write([tag, _time, record]).to_s
       }
     end
+
+    @d.events.each_with_index { |e, i|
+      orig = records[i]
+      assert_equal(orig[0], e[0])
+      assert_true(time <= e[1])
+      assert_equal(orig[2], e[2])
+    }
   end
 
   def test_message
-    d = create_driver
+    @d = create_driver
 
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
-
-    d.expect_emit "tag1", time, {"a"=>1}
-    d.expect_emit "tag2", time, {"a"=>2}
+    time = Fluent::EventTime.now
+    records = [
+      ["tag1", time, {"a" => 1}],
+      ["tag2", time, {"a" => 2}],
+    ]
 
-    d.run do
-      d.expected_emits.each {|tag,_time,record|
-        send_data Fluent::MessagePackFactory.msgpack_packer.write([tag, _time, record]).to_s
+    @d.run(expect_records: records.length, timeout: 5) do
+      records.each {|tag, _time, record|
+        send_data packer.write([tag, _time, record]).to_s
       }
     end
+
+    assert_equal(records, @d.events)
   end
 
   def test_forward
-    d = create_driver
+    @d = create_driver
 
     time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+    records = [
+      ["tag1", time, {"a" => 1}],
+      ["tag1", time, {"a" => 2}]
+    ]
 
-    d.expect_emit "tag1", time, {"a"=>1}
-    d.expect_emit "tag1", time, {"a"=>2}
-
-    d.run do
+    @d.run(expect_records: records.length, timeout: 20) do
       entries = []
-      d.expected_emits.each {|tag,_time,record|
+      records.each {|tag, _time, record|
         entries << [_time, record]
       }
-      send_data Fluent::MessagePackFactory.msgpack_packer.write(["tag1", entries]).to_s
+      send_data packer.write(["tag1", entries]).to_s
     end
+    assert_equal(records, @d.events)
   end
 
   def test_packed_forward
-    d = create_driver
+    @d = create_driver
 
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
-
-    d.expect_emit "tag1", time, {"a"=>1}
-    d.expect_emit "tag1", time, {"a"=>2}
+    time = Fluent::EventTime.now
+    records = [
+      ["tag1", time, {"a" => 1}],
+      ["tag1", time, {"a" => 2}],
+    ]
 
-    d.run do
+    @d.run(expect_records: records.length, timeout: 20) do
       entries = ''
-      d.expected_emits.each {|tag,_time,record|
-        Fluent::MessagePackFactory.msgpack_packer(entries).write([_time, record]).flush
+      records.each {|_tag, _time, record|
+        packer(entries).write([_time, record]).flush
       }
-      send_data Fluent::MessagePackFactory.msgpack_packer.write(["tag1", entries]).to_s
+      send_data packer.write(["tag1", entries]).to_s
     end
+    assert_equal(records, @d.events)
   end
 
   def test_message_json
-    d = create_driver
+    @d = create_driver
+
+    time = Fluent::EventTime.now
+    records = [
+      ["tag1", time, {"a" => 1}],
+      ["tag2", time, {"a" => 2}],
+    ]
+
+    @d.run(expect_records: records.length, timeout: 5) do
+      tag, _time, record = records[0]
+      send_data [tag, _time.to_i, record].to_json
+      tag, _time, record = records[1]
+      send_data [tag, _time.to_f, record].to_json
+    end
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    assert_equal(records, @d.events)
+  end
+
+  def test_message_with_tag
+    @d = create_driver(CONFIG + "tag new_tag")
 
-    d.expect_emit "tag1", time, {"a"=>1}
-    d.expect_emit "tag2", time, {"a"=>2}
+    time = Fluent::EventTime.now
+    records = [
+      ["tag1", time, {"a" => 1}],
+      ["tag2", time, {"a" => 2}],
+    ]
 
-    d.run do
-      d.expected_emits.each {|tag,_time,record|
-        send_data [tag, time, record].to_json
+    @d.run(expect_records: records.length, timeout: 5) do
+      records.each {|tag, _time, record|
+        send_data packer.write([tag, _time, record]).to_s
       }
     end
-  end
 
-  def create_driver(klass, conf)
-    Fluent::Test::InputTestDriver.new(klass).configure(conf)
+    @d.events.each { |event|
+      assert_equal("new_tag", event[0])
+    }
   end
 
-  def send_data(data)
-    io = connect
-    begin
-      io.write data
-    ensure
-      io.close
+  data('string chunk' => 'broken string',
+       'integer chunk' => 10)
+  def test_broken_message(data)
+    @d = create_driver
+    @d.run(shutdown: false, timeout: 5) do
+      @d.instance.__send__(:on_message, data)
     end
-  end
-end
-
-class UnixInputTest < Test::Unit::TestCase
-  include StreamInputTest
-
-  TMP_DIR = File.dirname(__FILE__) + "/../tmp/in_unix#{ENV['TEST_ENV_NUMBER']}"
-  CONFIG = %[
-    path #{TMP_DIR}/unix
-    backlog 1000
-  ]
 
-  def create_driver(conf=CONFIG)
-    super(Fluent::UnixInput, conf)
-  end
-
-  def test_configure
-    d = create_driver
-    assert_equal "#{TMP_DIR}/unix", d.instance.path
-    assert_equal 1000, d.instance.backlog
-  end
+    assert_equal 0, @d.events.size
 
-  def connect
-    UNIXSocket.new("#{TMP_DIR}/unix")
+    logs = @d.instance.log.logs
+    assert_equal 1, logs.select { |line|
+      line =~ / \[warn\]: incoming data is broken: msg=#{data.inspect}/
+    }.size, "should not accept broken chunk"
   end
 end unless Fluent.windows?