fluentd 1.16.8-x86-mingw32 → 1.17.0-x86-mingw32

data/test/plugin/test_output.rb

@@ -1111,7 +1111,7 @@ class OutputTest < Test::Unit::TestCase
         config: config_element(
           "ROOT", "", {},
           [
-            config_element("buffer", "", {}),
+            config_element("buffer", "", {}), 
             config_element("secondary", "", {"@type" => "test", "name" => "test"}),
           ]
         ),

data/test/plugin/test_output_as_buffered.rb

@@ -1343,7 +1343,6 @@ class BufferedOutputTest < Test::Unit::TestCase
       hash = {
         'flush_interval' => 10,
         'flush_thread_count' => 1,
-        'flush_thread_interval' => 0.1,
         'flush_thread_burst_interval' => 0.1,
         'chunk_limit_size' => 1024,
       }
@@ -1446,7 +1445,7 @@ class BufferedOutputTest < Test::Unit::TestCase
 
       assert{ @i.buffer.stage.size == 3 }
 
-      # to trigger try_flush with flush_thread_interval
+      # to trigger try_flush with flush_thread_burst_interval
       Timecop.freeze( Time.parse('2016-04-13 14:04:11 +0900') )
       @i.enqueue_thread_wait
       Timecop.freeze( Time.parse('2016-04-13 14:04:12 +0900') )
@@ -1455,6 +1454,7 @@ class BufferedOutputTest < Test::Unit::TestCase
       @i.enqueue_thread_wait
       Timecop.freeze( Time.parse('2016-04-13 14:04:14 +0900') )
       @i.enqueue_thread_wait
+      @i.flush_thread_wakeup
 
       assert{ @i.buffer.stage.size == 0 }
 

data/test/plugin/test_output_as_buffered_retries.rb

@@ -941,7 +941,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       @i.enqueue_thread_wait
 
       @i.flush_thread_wakeup
-      waiting(4){ Thread.pass until @i.write_count > 0 && @i.num_errors > 0 }
+      waiting(4){ Thread.pass until @i.write_count > 0 }
       waiting(4) do
         state = @i.instance_variable_get(:@output_flush_threads).first
         state.thread.status == 'sleep'
@@ -953,7 +953,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       now = @i.next_flush_time
       Timecop.freeze( now )
       @i.flush_thread_wakeup
-      waiting(4){ Thread.pass until @i.write_count > 1 && @i.num_errors > 1 }
+      waiting(4){ Thread.pass until @i.write_count > 1 }
       waiting(4) do
         state = @i.instance_variable_get(:@output_flush_threads).first
         state.thread.status == 'sleep'

data/test/plugin/test_owned_by.rb

@@ -26,7 +26,6 @@ class OwnedByMixinTest < Test::Unit::TestCase
 
       assert_equal parent.object_id, child.owner.object_id
 
-      assert child.instance_eval{ @_plugin_id_configured }
       assert_equal 'my_parent_id', child.instance_eval{ @_plugin_id }
 
       assert_equal Fluent::Log::LEVEL_TRACE, child.log.level

data/test/plugin/test_parser_csv.rb

@@ -164,7 +164,7 @@ class CSVParserTest < ::Test::Unit::TestCase
       text = 'a"b,"a"""c"'
       assert_raise(CSV::MalformedCSVError) {
         normal.instance.parse(text) { |t, r| }
-      }
+      } 
       assert_nothing_raised {
         # generate broken record
         fast.instance.parse(text) { |t, r| }

data/test/plugin/test_parser_json.rb

@@ -8,37 +8,6 @@ class JsonParserTest < ::Test::Unit::TestCase
     @parser = Fluent::Test::Driver::Parser.new(Fluent::Plugin::JSONParser)
   end
 
-  sub_test_case "configure_json_parser" do
-    data("oj", [:oj, [Oj.method(:load), Oj::ParseError]])
-    data("json", [:json, [JSON.method(:load), JSON::ParserError]])
-    data("yajl", [:yajl, [Yajl.method(:load), Yajl::ParseError]])
-    def test_return_each_loader((input, expected_return))
-      result = @parser.instance.configure_json_parser(input)
-      assert_equal expected_return, result
-    end
-
-    def test_raise_exception_for_unknown_input
-      assert_raise RuntimeError do
-        @parser.instance.configure_json_parser(:unknown)
-      end
-    end
-
-    def test_fall_back_oj_to_yajl_if_oj_not_available
-      stub(Fluent::OjOptions).available? { false }
-
-      result = @parser.instance.configure_json_parser(:oj)
-
-      assert_equal [Yajl.method(:load), Yajl::ParseError], result
-      logs = @parser.logs.collect do |log|
-        log.gsub(/\A\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4} /, "")
-      end
-      assert_equal(
-        ["[info]: Oj is not installed, and failing back to Yajl for json parser\n"],
-        logs
-      )
-    end
-  end
-
   data('oj' => 'oj', 'yajl' => 'yajl')
   def test_parse(data)
     @parser.configure('json_parser' => data)
@@ -166,4 +135,110 @@ class JsonParserTest < ::Test::Unit::TestCase
       end
     end
   end
+
+  sub_test_case "various record pattern" do
+    data("Only string", { record: '"message"', expected: [nil] }, keep: true)
+    data("Only string without quotation", { record: "message", expected: [nil] }, keep: true)
+    data("Only number", { record: "0", expected: [nil] }, keep: true)
+    data(
+      "Array of Hash",
+      {
+        record: '[{"k1": 1}, {"k2": 2}]',
+        expected: [{"k1" => 1}, {"k2" => 2}]
+      },
+      keep: true,
+    )
+    data(
+      "Array of both Hash and invalid",
+      {
+        record: '[{"k1": 1}, "string", {"k2": 2}, 0]',
+        expected: [{"k1" => 1}, nil, {"k2" => 2}, nil]
+      },
+      keep: true,
+    )
+    data(
+      "Array of all invalid",
+      {
+        record: '["string", 0, [{"k": 0}]]',
+        expected: [nil, nil, nil]
+      },
+      keep: true,
+    )
+
+    def test_oj(data)
+      parsed_records = []
+      @parser.configure("json_parser" => "oj")
+      @parser.instance.parse(data[:record]) { |time, record|
+        parsed_records.append(record)
+      }
+      assert_equal(data[:expected], parsed_records)
+    end
+
+    def test_yajl(data)
+      parsed_records = []
+      @parser.configure("json_parser" => "yajl")
+      @parser.instance.parse(data[:record]) { |time, record|
+        parsed_records.append(record)
+      }
+      assert_equal(data[:expected], parsed_records)
+    end
+
+    def test_json(json)
+      parsed_records = []
+      @parser.configure("json_parser" => "json")
+      @parser.instance.parse(data[:record]) { |time, record|
+        parsed_records.append(record)
+      }
+      assert_equal(data[:expected], parsed_records)
+    end
+  end
+
+  # This becomes NoMethodError if a non-Hash object is passed to convert_values.
+  # https://github.com/fluent/fluentd/issues/4100
+  sub_test_case "execute_convert_values with null_empty_string" do
+    data("Only string", { record: '"message"', expected: [nil] }, keep: true)
+    data(
+      "Hash",
+      {
+        record: '{"k1": 1, "k2": ""}',
+        expected: [{"k1" => 1, "k2" => nil}]
+      },
+      keep: true,
+    )
+    data(
+      "Array of Hash",
+      {
+        record: '[{"k1": 1}, {"k2": ""}]',
+        expected: [{"k1" => 1}, {"k2" => nil}]
+      },
+      keep: true,
+    )
+
+    def test_oj(data)
+      parsed_records = []
+      @parser.configure("json_parser" => "oj", "null_empty_string" => true)
+      @parser.instance.parse(data[:record]) { |time, record|
+        parsed_records.append(record)
+      }
+      assert_equal(data[:expected], parsed_records)
+    end
+
+    def test_yajl(data)
+      parsed_records = []
+      @parser.configure("json_parser" => "yajl", "null_empty_string" => true)
+      @parser.instance.parse(data[:record]) { |time, record|
+        parsed_records.append(record)
+      }
+      assert_equal(data[:expected], parsed_records)
+    end
+
+    def test_json(json)
+      parsed_records = []
+      @parser.configure("json_parser" => "json", "null_empty_string" => true)
+      @parser.instance.parse(data[:record]) { |time, record|
+        parsed_records.append(record)
+      }
+      assert_equal(data[:expected], parsed_records)
+    end
+  end
 end

data/test/plugin/test_parser_msgpack.rb

@@ -0,0 +1,127 @@
+require_relative '../helper'
+require 'fluent/test/driver/parser'
+require 'fluent/plugin/parser_msgpack'
+
+class MessagePackParserTest < ::Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Parser.new(Fluent::Plugin::MessagePackParser).configure(conf)
+  end
+
+  sub_test_case "simple setting" do
+    data(
+      "Normal Hash",
+      {
+        input: "\x82\xA7message\xADHello msgpack\xA3numd",
+        expected: [{"message" => "Hello msgpack", "num" => 100}]
+      },
+      keep: true
+    )
+    data(
+      "Array of multiple Hash",
+      {
+        input: "\x92\x81\xA7message\xA3foo\x81\xA7message\xA3bar",
+        expected: [{"message"=>"foo"}, {"message"=>"bar"}]
+      },
+      keep: true
+    )
+    data(
+      "String",
+      {
+        # "Hello msgpack".to_msgpack
+        input: "\xADHello msgpack",
+        expected: [nil]
+      },
+      keep: true
+    )
+    data(
+      "Array of String",
+      {
+        # ["foo", "bar"].to_msgpack
+        input: "\x92\xA3foo\xA3bar",
+        expected: [nil, nil]
+      },
+      keep: true
+    )
+    data(
+      "Array of String and Hash",
+      {
+        # ["foo", {message: "bar"}].to_msgpack
+        input: "\x92\xA3foo\x81\xA7message\xA3bar",
+        expected: [nil, {"message"=>"bar"}]
+      },
+      keep: true
+    )
+
+    def test_parse(data)
+      parsed_records = []
+      create_driver("").instance.parse(data[:input]) do |time, record|
+        parsed_records.append(record)
+      end
+      assert_equal(data[:expected], parsed_records)
+    end
+
+    def test_parse_io(data)
+      parsed_records = []
+      StringIO.open(data[:input]) do |io|
+        create_driver("").instance.parse_io(io) do |time, record|
+          parsed_records.append(record)
+        end
+      end
+      assert_equal(data[:expected], parsed_records)
+    end
+  end
+
+  # This becomes NoMethodError if a non-Hash object is passed to convert_values.
+  # https://github.com/fluent/fluentd/issues/4100
+  sub_test_case "execute_convert_values with null_empty_string" do
+    data(
+      "Normal hash",
+      {
+        # {message: "foo", empty: ""}.to_msgpack
+        input: "\x82\xA7message\xA3foo\xA5empty\xA0",
+        expected: [{"message" => "foo", "empty" => nil}]
+      },
+      keep: true
+    )
+    data(
+      "Array of multiple Hash",
+      {
+        # [{message: "foo", empty: ""}, {message: "bar", empty: ""}].to_msgpack
+        input: "\x92\x82\xA7message\xA3foo\xA5empty\xA0\x82\xA7message\xA3bar\xA5empty\xA0",
+        expected: [{"message"=>"foo", "empty" => nil}, {"message"=>"bar", "empty" => nil}]
+      },
+      keep: true
+    )
+    data(
+      "String",
+      {
+        # "Hello msgpack".to_msgpack
+        input: "\xADHello msgpack",
+        expected: [nil]
+      },
+      keep: true
+    )
+
+    def test_parse(data)
+      parsed_records = []
+      create_driver("null_empty_string").instance.parse(data[:input]) do |time, record|
+        parsed_records.append(record)
+      end
+      assert_equal(data[:expected], parsed_records)
+    end
+
+    def test_parse_io(data)
+      parsed_records = []
+      StringIO.open(data[:input]) do |io|
+        create_driver("null_empty_string").instance.parse_io(io) do |time, record|
+          parsed_records.append(record)
+        end
+      end
+      assert_equal(data[:expected], parsed_records)
+    end
+  end
+end

data/test/plugin/test_storage.rb

@@ -68,7 +68,6 @@ class StorageTest < Test::Unit::TestCase
 
       assert_equal 'mytest', s.owner.system_config.process_name
       assert_equal '1', s.instance_eval{ @_plugin_id }
-      assert_equal true, s.instance_eval{ @_plugin_id_configured }
     end
 
     test 'does NOT have features for high-performance/high-consistent storages' do

data/test/plugin_helper/test_child_process.rb

@@ -569,7 +569,7 @@ class ChildProcessTest < Test::Unit::TestCase
   unless Fluent.windows?
     test 'can specify subprocess name' do
       io = IO.popen([["cat", "caaaaaaaaaaat"], '-'])
-      process_naming_enabled = (open("|ps opid,cmd"){|_io| _io.readlines }.count{|line| line.include?("caaaaaaaaaaat") } > 0)
+      process_naming_enabled = (IO.popen(["ps", "opid,cmd"]){|_io| _io.readlines }.count{|line| line.include?("caaaaaaaaaaat") } > 0)
       Process.kill(:TERM, io.pid) rescue nil
       io.close rescue nil
 
@@ -586,7 +586,7 @@ class ChildProcessTest < Test::Unit::TestCase
           m.lock
           ran = true
           pids << @d.child_process_id
-          proc_lines += open("|ps opid,cmd"){|_io| _io.readlines }
+          proc_lines += IO.popen(["ps", "opid,cmd"]){|_io| _io.readlines }
           m.unlock
           readio.read
         end
@@ -645,8 +645,8 @@ class ChildProcessTest < Test::Unit::TestCase
   unless Fluent.windows?
     test 'can change working directory' do
       # check my real /tmp directory (for mac)
-      cmd = %[|ruby -e 'Dir.chdir("/tmp"); puts Dir.pwd']
-      mytmpdir = open(cmd){|io| io.read.chomp }
+      cmd = ['ruby', '-e', 'Dir.chdir("/tmp"); puts Dir.pwd']
+      mytmpdir = IO.popen(cmd){|io| io.read.chomp }
 
       m = Mutex.new
       str = nil

data/test/plugin_helper/test_http_server_helper.rb

@@ -14,7 +14,7 @@ class HttpHelperTest < Test::Unit::TestCase
   CERT_CA_DIR = File.expand_path(File.dirname(__FILE__) + '/data/cert/with_ca')
 
   def setup
-    @port = unused_port(protocol: :tcp)
+    @port = unused_port
   end
 
   def teardown