fluentd 1.10.1-x86-mingw32 → 1.11.1-x86-mingw32

data/test/plugin/test_in_gc_stat.rb

@@ -22,9 +22,14 @@ class GCStatInputTest < Test::Unit::TestCase
     assert_equal("t1", d.instance.tag)
   end
 
-  def test_emit
+  def setup_gc_stat
     stat = GC.stat
     stub(GC).stat { stat }
+    stat
+  end
+
+  def test_emit
+    stat = setup_gc_stat
 
     d = create_driver
     d.run(expect_emits: 2)
@@ -36,4 +41,22 @@ class GCStatInputTest < Test::Unit::TestCase
       assert(events[i][1].is_a?(Fluent::EventTime))
     }
   end
+
+  def test_emit_with_use_symbol_keys_false
+    stat = setup_gc_stat
+    result = {}
+    stat.each_pair { |k, v|
+      result[k.to_s] = v
+    }
+
+    d = create_driver(CONFIG + "use_symbol_keys false")
+    d.run(expect_emits: 2)
+
+    events = d.events
+    assert(events.length > 0)
+    events.each_index {|i|
+      assert_equal(result, events[i][2])
+      assert(events[i][1].is_a?(Fluent::EventTime))
+    }
+  end
 end

data/test/plugin/test_in_http.rb

@@ -116,6 +116,36 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal_event_time time, d.events[1][1]
   end
 
+  data('json' => ['json', :to_json],
+       'msgpack' => ['msgpack', :to_msgpack])
+  def test_default_with_time_format(data)
+    param, method_name = data
+    d = create_driver(CONFIG + %[
+      <parse>
+        keep_time_key
+        time_format %iso8601
+      </parse>
+    ])
+
+    time = event_time("2020-06-10T01:14:27+00:00")
+    events = [
+      ["tag1", time, {"a" => 1, "time" => '2020-06-10T01:14:27+00:00'}],
+      ["tag2", time, {"a" => 2, "time" => '2020-06-10T01:14:27+00:00'}],
+    ]
+    res_codes = []
+
+    d.run(expect_records: 2) do
+      events.each do |tag, t, record|
+        res = post("/#{tag}", {param => record.__send__(method_name)})
+        res_codes << res.code
+      end
+    end
+    assert_equal ["200", "200"], res_codes
+    assert_equal events, d.events
+    assert_equal_event_time time, d.events[0][1]
+    assert_equal_event_time time, d.events[1][1]
+  end
+
   def test_multi_json
     d = create_driver
     time = event_time("2011-01-02 13:14:15 UTC")
@@ -163,6 +193,33 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal_event_time time, d.events[1][1]
   end
 
+  data('json' => ['json', :to_json],
+       'msgpack' => ['msgpack', :to_msgpack])
+  def test_default_multi_with_time_format(data)
+    param, method_name = data
+    d = create_driver(CONFIG + %[
+      <parse>
+        keep_time_key
+        time_format %iso8601
+      </parse>
+    ])
+    time = event_time("2020-06-10T01:14:27+00:00")
+    events = [
+      ["tag1", time, {'a' => 1, 'time' => "2020-06-10T01:14:27+00:00"}],
+      ["tag1", time, {'a' => 2, 'time' => "2020-06-10T01:14:27+00:00"}],
+    ]
+    tag = "tag1"
+    res_codes = []
+    d.run(expect_records: 2, timeout: 5) do
+      res = post("/#{tag}", {param => events.map { |e| e[2] }.__send__(method_name)})
+      res_codes << res.code
+    end
+    assert_equal ["200"], res_codes
+    assert_equal events, d.events
+    assert_equal_event_time time, d.events[0][1]
+    assert_equal_event_time time, d.events[1][1]
+  end
+
   def test_multi_json_with_nonexistent_time_key
     d = create_driver(CONFIG + %[
       <parse>

data/test/plugin/test_in_syslog.rb

@@ -375,7 +375,7 @@ EOS
       ]
       msgs.each { |msg|
         m = msg['msg']
-        msg['msg'] = "#{m.size + 1} #{m}"
+        msg['msg'] = "#{m.size} #{m}"
       }
       msgs
     end

data/test/plugin/test_in_tail.rb

@@ -23,9 +23,6 @@ class TailInputTest < Test::Unit::TestCase
   def cleanup_directory(path)
     FileUtils.rm_rf(path, secure: true)
     if File.exist?(path)
-      # ensure files are closed for Windows, on which deleted files
-      # are still visible from filesystem
-      GC.start(full_mark: true, immediate_mark: true, immediate_sweep: true)
       FileUtils.remove_entry_secure(path, true)
     end
     FileUtils.mkdir_p(path)
@@ -105,6 +102,12 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal ["tail.txt", "test2", "tmp,dev"], d.instance.paths
     end
 
+    test "multi paths with same path configured twice" do
+      c = config_element("ROOT", "", { "path" => "test1.txt,test2.txt,test1.txt", "tag" => "t1", "path_delimiter" => "," })
+      d = create_driver(c + PARSE_SINGLE_LINE_CONFIG, false)
+      assert_equal ["test2.txt","test1.txt"].sort, d.instance.paths.sort
+    end
+
     test "multi paths with invaid path_delimiter" do
       c = config_element("ROOT", "", { "path" => "tail.txt|test2|tmp,dev", "tag" => "t1", "path_delimiter" => "*" })
       assert_raise(Fluent::ConfigError) do
@@ -1025,6 +1028,17 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal EX_PATHS - [EX_PATHS.last], plugin.expand_paths.sort
     end
 
+    def test_expand_paths_with_duplicate_configuration
+      expanded_paths = [
+        'test/plugin/data/log/foo/bar.log',
+        'test/plugin/data/log/test.log'
+      ]
+      duplicate_config = EX_CONFIG.dup
+      duplicate_config["path"]="test/plugin/data/log/**/*.log, test/plugin/data/log/**/*.log"
+      plugin = create_driver(EX_CONFIG, false).instance
+      assert_equal expanded_paths, plugin.expand_paths.sort
+    end
+
     def test_expand_paths_with_timezone
       ['Asia/Taipei', '+08'].each do |tz_type|
         taipei_config = EX_CONFIG + config_element("", "", {"path_timezone" => tz_type})
@@ -1065,9 +1079,6 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal expected_files, plugin.expand_paths.sort
     end
 
-    # For https://github.com/fluent/fluentd/issues/1455
-    # This test is fragile because test content depends on internal implementation.
-    # So if you modify in_tail internal, this test may break.
     def test_unwatched_files_should_be_removed
       config = config_element("", "", {
                                 "tag" => "tail",
@@ -1079,22 +1090,15 @@ class TailInputTest < Test::Unit::TestCase
                               })
       d = create_driver(config, false)
       d.end_if { d.instance.instance_variable_get(:@tails).keys.size >= 1 }
-      d.run(expect_emits: 1, shutdown: false, timeout: 1) do
+      d.run(expect_emits: 1, shutdown: false) do
         File.open("#{TMP_DIR}/tail.txt", "ab") { |f| f.puts "test3\n" }
       end
 
       cleanup_directory(TMP_DIR)
       waiting(20) { sleep 0.1 until Dir.glob("#{TMP_DIR}/*.txt").size == 0 } # Ensure file is deleted on Windows
-      waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size == 0 }
+      waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size <= 0 }
 
-      # Previous implementation has an infinite watcher creation bug.
-      # Following code checks such unexpected bug by counting actual object allocation.
-      base_num = count_timer_object
-      2.times {
-        sleep 1
-        num = count_timer_object
-        assert_equal base_num, num
-      }
+      assert_equal 0, d.instance.instance_variable_get(:@tails).keys.size
 
       d.instance_shutdown
     end

data/test/plugin/test_in_unix.rb

@@ -1,126 +1,181 @@
 require_relative '../helper'
-require 'fluent/test'
+require 'fluent/test/driver/input'
 require 'fluent/plugin/in_unix'
 
-module StreamInputTest
+class UnixInputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
+    @d = nil
   end
 
-  def test_time
-    d = create_driver
+  def teardown
+    @d.instance_shutdown if @d
+  end
 
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
-    Fluent::Engine.now = time
+  TMP_DIR = File.dirname(__FILE__) + "/../tmp/in_unix#{ENV['TEST_ENV_NUMBER']}"
+  CONFIG = %[
+    path #{TMP_DIR}/unix
+    backlog 1000
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::UnixInput).configure(conf)
+  end
 
-    d.expect_emit "tag1", time, {"a"=>1}
-    d.expect_emit "tag2", time, {"a"=>2}
+  def packer(*args)
+    Fluent::MessagePackFactory.msgpack_packer(*args)
+  end
 
-    d.run do
-      d.expected_emits.each {|tag,_time,record|
-        send_data Fluent::MessagePackFactory.msgpack_packer.write([tag, 0, record]).to_s
+  def unpacker
+    Fluent::MessagePackFactory.msgpack_unpacker
+  end
+
+  def send_data(data)
+    io = UNIXSocket.new("#{TMP_DIR}/unix")
+    begin
+      io.write data
+    ensure
+      io.close
+    end
+  end
+
+  def test_configure
+    @d = create_driver
+    assert_equal "#{TMP_DIR}/unix", @d.instance.path
+    assert_equal 1000, @d.instance.backlog
+  end
+
+  def test_time
+    @d = create_driver
+
+    time = Fluent::EventTime.now
+    records = [
+      ["tag1", 0, {"a" => 1}],
+      ["tag2", nil, {"a" => 2}],
+    ]
+
+    @d.run(expect_records: records.length, timeout: 5) do
+      records.each {|tag, _time, record|
+        send_data packer.write([tag, _time, record]).to_s
       }
     end
+
+    @d.events.each_with_index { |e, i|
+      orig = records[i]
+      assert_equal(orig[0], e[0])
+      assert_true(time <= e[1])
+      assert_equal(orig[2], e[2])
+    }
   end
 
   def test_message
-    d = create_driver
+    @d = create_driver
 
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
-
-    d.expect_emit "tag1", time, {"a"=>1}
-    d.expect_emit "tag2", time, {"a"=>2}
+    time = Fluent::EventTime.now
+    records = [
+      ["tag1", time, {"a" => 1}],
+      ["tag2", time, {"a" => 2}],
+    ]
 
-    d.run do
-      d.expected_emits.each {|tag,_time,record|
-        send_data Fluent::MessagePackFactory.msgpack_packer.write([tag, _time, record]).to_s
+    @d.run(expect_records: records.length, timeout: 5) do
+      records.each {|tag, _time, record|
+        send_data packer.write([tag, _time, record]).to_s
       }
     end
+
+    assert_equal(records, @d.events)
   end
 
   def test_forward
-    d = create_driver
+    @d = create_driver
 
     time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+    records = [
+      ["tag1", time, {"a" => 1}],
+      ["tag1", time, {"a" => 2}]
+    ]
 
-    d.expect_emit "tag1", time, {"a"=>1}
-    d.expect_emit "tag1", time, {"a"=>2}
-
-    d.run do
+    @d.run(expect_records: records.length, timeout: 20) do
       entries = []
-      d.expected_emits.each {|tag,_time,record|
+      records.each {|tag, _time, record|
         entries << [_time, record]
       }
-      send_data Fluent::MessagePackFactory.msgpack_packer.write(["tag1", entries]).to_s
+      send_data packer.write(["tag1", entries]).to_s
     end
+    assert_equal(records, @d.events)
   end
 
   def test_packed_forward
-    d = create_driver
+    @d = create_driver
 
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
-
-    d.expect_emit "tag1", time, {"a"=>1}
-    d.expect_emit "tag1", time, {"a"=>2}
+    time = Fluent::EventTime.now
+    records = [
+      ["tag1", time, {"a" => 1}],
+      ["tag1", time, {"a" => 2}],
+    ]
 
-    d.run do
+    @d.run(expect_records: records.length, timeout: 20) do
       entries = ''
-      d.expected_emits.each {|tag,_time,record|
-        Fluent::MessagePackFactory.msgpack_packer(entries).write([_time, record]).flush
+      records.each {|_tag, _time, record|
+        packer(entries).write([_time, record]).flush
       }
-      send_data Fluent::MessagePackFactory.msgpack_packer.write(["tag1", entries]).to_s
+      send_data packer.write(["tag1", entries]).to_s
     end
+    assert_equal(records, @d.events)
   end
 
   def test_message_json
-    d = create_driver
+    @d = create_driver
+
+    time = Fluent::EventTime.now
+    records = [
+      ["tag1", time, {"a" => 1}],
+      ["tag2", time, {"a" => 2}],
+    ]
+
+    @d.run(expect_records: records.length, timeout: 5) do
+      tag, _time, record = records[0]
+      send_data [tag, _time.to_i, record].to_json
+      tag, _time, record = records[1]
+      send_data [tag, _time.to_f, record].to_json
+    end
 
-    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    assert_equal(records, @d.events)
+  end
+
+  def test_message_with_tag
+    @d = create_driver(CONFIG + "tag new_tag")
 
-    d.expect_emit "tag1", time, {"a"=>1}
-    d.expect_emit "tag2", time, {"a"=>2}
+    time = Fluent::EventTime.now
+    records = [
+      ["tag1", time, {"a" => 1}],
+      ["tag2", time, {"a" => 2}],
+    ]
 
-    d.run do
-      d.expected_emits.each {|tag,_time,record|
-        send_data [tag, time, record].to_json
+    @d.run(expect_records: records.length, timeout: 5) do
+      records.each {|tag, _time, record|
+        send_data packer.write([tag, _time, record]).to_s
       }
     end
-  end
 
-  def create_driver(klass, conf)
-    Fluent::Test::InputTestDriver.new(klass).configure(conf)
+    @d.events.each { |event|
+      assert_equal("new_tag", event[0])
+    }
   end
 
-  def send_data(data)
-    io = connect
-    begin
-      io.write data
-    ensure
-      io.close
+  data('string chunk' => 'broken string',
+       'integer chunk' => 10)
+  def test_broken_message(data)
+    @d = create_driver
+    @d.run(shutdown: false, timeout: 5) do
+      @d.instance.__send__(:on_message, data)
     end
-  end
-end
-
-class UnixInputTest < Test::Unit::TestCase
-  include StreamInputTest
-
-  TMP_DIR = File.dirname(__FILE__) + "/../tmp/in_unix#{ENV['TEST_ENV_NUMBER']}"
-  CONFIG = %[
-    path #{TMP_DIR}/unix
-    backlog 1000
-  ]
 
-  def create_driver(conf=CONFIG)
-    super(Fluent::UnixInput, conf)
-  end
-
-  def test_configure
-    d = create_driver
-    assert_equal "#{TMP_DIR}/unix", d.instance.path
-    assert_equal 1000, d.instance.backlog
-  end
+    assert_equal 0, @d.events.size
 
-  def connect
-    UNIXSocket.new("#{TMP_DIR}/unix")
+    logs = @d.instance.log.logs
+    assert_equal 1, logs.select { |line|
+      line =~ / \[warn\]: incoming data is broken: msg=#{data.inspect}/
+    }.size, "should not accept broken chunk"
   end
 end unless Fluent.windows?

data/test/plugin/test_out_forward.rb

@@ -1056,7 +1056,7 @@ EOL
         e = assert_raise Fluent::UnrecoverableError do
           d.instance_start
         end
-        assert_match(/Failed to establish connection/, e.message)
+        assert_match(/failed to establish connection/, e.message)
       end
     end
 
@@ -1092,7 +1092,7 @@ EOL
           d.instance_start
         end
 
-        assert_match(/Failed to establish connection/, e.message)
+        assert_match(/failed to establish connection/, e.message)
       end
     end
 
@@ -1200,6 +1200,15 @@ EOL
       end
     end
 
+    test 'create timer of purging obsolete sockets' do
+      output_conf = CONFIG + %[keepalive true]
+      d = create_driver(output_conf)
+
+      mock(d.instance).timer_execute(:out_forward_heartbeat_request, 1).once
+      mock(d.instance).timer_execute(:out_forward_keep_alived_socket_watcher, 5).once
+      d.instance_start
+    end
+
     sub_test_case 'with require_ack_response' do
       test 'Create connection per send_data' do
         target_input_driver = create_target_input_driver(conf: TARGET_CONFIG)