fluentd 1.10.2-x64-mingw32 → 1.11.2-x64-mingw32

data/lib/fluent/plugin_helper/child_process.rb

@@ -259,6 +259,9 @@ module Fluent
 
         if !mode.include?(:stderr) && !mode.include?(:read_with_stderr)
           spawn_opts[:err] = IO::NULL if stderr == :discard
+          if !mode.include?(:read) && !mode.include?(:read_with_stderr)
+            spawn_opts[:out] = IO::NULL
+          end
           writeio, readio, wait_thread = *Open3.popen2(*spawn_args, spawn_opts)
         elsif mode.include?(:read_with_stderr)
           writeio, readio, wait_thread = *Open3.popen2e(*spawn_args, spawn_opts)
@@ -275,8 +278,6 @@ module Fluent
         if mode.include?(:read) || mode.include?(:read_with_stderr)
           readio.set_encoding(external_encoding, internal_encoding, **encoding_options)
           readio_in_use = true
-        else
-          readio.reopen(IO::NULL) if readio
         end
         if mode.include?(:stderr)
           stderrio.set_encoding(external_encoding, internal_encoding, **encoding_options)

data/lib/fluent/plugin_helper/record_accessor.rb

@@ -41,9 +41,11 @@ module Fluent
             else
               mcall = method(:call_dig)
               mdelete = method(:delete_nest)
+              mset = method(:set_nest)
               singleton_class.module_eval do
                 define_method(:call, mcall)
                 define_method(:delete, mdelete)
+                define_method(:set, mset)
               end
             end
           end
@@ -75,6 +77,18 @@ module Fluent
           nil
         end
 
+        def set(r, v)
+          r[@keys] = v
+        end
+
+        # set_nest doesn't create intermediate object. If key doesn't exist, no effect.
+        # See also: https://bugs.ruby-lang.org/issues/11747
+        def set_nest(r, v)
+          r.dig(*@dig_keys)&.[]=(@last_key, v)
+        rescue
+          nil
+        end
+
         def self.parse_parameter(param)
           if param.start_with?('$.')
             parse_dot_notation(param)

data/lib/fluent/plugin_helper/service_discovery.rb

@@ -43,6 +43,13 @@ module Fluent
         super
       end
 
+      %i[after_start stop before_shutdown shutdown after_shutdown close terminate].each do |mth|
+        define_method(mth) do
+          @discovery_manager&.__send__(mth)
+          super()
+        end
+      end
+
       private
 
       # @param title [Symbol] the thread name. this value should be unique.

data/lib/fluent/plugin_helper/service_discovery/manager.rb

@@ -60,6 +60,14 @@ module Fluent
           end
         end
 
+        %i[after_start stop before_shutdown shutdown after_shutdown close terminate].each do |mth|
+          define_method(mth) do
+            @discoveries.each do |d|
+              d.__send__(mth)
+            end
+          end
+        end
+
         def run_once
           # Don't care race in this loop intentionally
           s = @queue.size

data/lib/fluent/plugin_helper/socket.rb

@@ -199,7 +199,7 @@ module Fluent
         list = []
         data.scan(pattern) { |match| list << OpenSSL::X509::Certificate.new(match) }
         if list.length == 0
-          log.warn "cert_path does not contain a valid certificate"
+          raise Fluent::ConfigError, "cert_path does not contain a valid certificate"
         end
         list
       end

data/lib/fluent/plugin_helper/socket_option.rb

@@ -67,8 +67,8 @@ module Fluent
           end
         else
           if linger_timeout
-          optval = [1, linger_timeout.to_i].pack(FORMAT_STRUCT_LINGER)
-          socket_option_set_one(sock, :SO_LINGER, optval)
+            optval = [1, linger_timeout.to_i].pack(FORMAT_STRUCT_LINGER)
+            socket_option_set_one(sock, :SO_LINGER, optval)
           end
         end
         if recv_timeout

data/lib/fluent/supervisor.rb

@@ -556,7 +556,7 @@ module Fluent
       end
 
       if dry_run
-        $log.info 'finsihed dry run mode'
+        $log.info 'finished dry run mode'
         exit 0
       else
         supervise
@@ -593,7 +593,10 @@ module Fluent
 
       main_process do
         create_socket_manager if @standalone_worker
-        ServerEngine::Privilege.change(@chuser, @chgroup) if @standalone_worker
+        if @standalone_worker
+          ServerEngine::Privilege.change(@chuser, @chgroup)
+          File.umask(0)
+        end
         MessagePackFactory.init(enable_time_support: @system_config.enable_msgpack_time_support)
         Fluent::Engine.init(@system_config)
         Fluent::Engine.run_configure(@conf)
@@ -801,7 +804,7 @@ module Fluent
             Fluent::Engine.reload_config(conf)
           end
         rescue => e
-          # it is guranteed that config file is valid by supervisor side. but it's not atomic becuase of using signals to commnicate between worker and super
+          # it is guaranteed that config file is valid by supervisor side. but it's not atomic because of using signals to commnicate between worker and super
           # So need this rescue code
           $log.error("failed to reload config: #{e}")
           next

data/lib/fluent/test/filter_test.rb

@@ -30,12 +30,12 @@ module Fluent
       attr_reader :filtered
       attr_accessor :tag
 
-      def emit(record, time = Engine.now)
+      def emit(record, time = EventTime.now)
         emit_with_tag(@tag, record, time)
       end
       alias_method :filter, :emit
 
-      def emit_with_tag(tag, record, time = Engine.now)
+      def emit_with_tag(tag, record, time = EventTime.now)
         @events[tag] ||= MultiEventStream.new
         @events[tag].add(time, record)
       end

data/lib/fluent/test/output_test.rb

@@ -41,7 +41,7 @@ module Fluent
 
       attr_accessor :tag
 
-      def emit(record, time=Engine.now)
+      def emit(record, time=EventTime.now)
         es = OneEventStream.new(time, record)
         @instance.emit_events(@tag, es)
       end
@@ -62,7 +62,7 @@ module Fluent
 
       attr_accessor :tag
 
-      def emit(record, time=Engine.now)
+      def emit(record, time=EventTime.now)
         @entries << [time, record]
         self
       end
@@ -110,7 +110,7 @@ module Fluent
 
       attr_accessor :tag
 
-      def emit(record, time=Engine.now)
+      def emit(record, time=EventTime.now)
         @entries << [time, record]
         self
       end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.10.2'
+  VERSION = '1.11.2'
 
 end

data/test/plugin/out_forward/test_load_balancer.rb

@@ -49,6 +49,44 @@ class LoadBalancerTest < Test::Unit::TestCase
       end
     end
 
+    test 'call like round robin without weight=0 node' do
+      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
+      n1 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
+      n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
+      n3 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 0)
+
+      lb.rebuild_weight_array([n1, n2, n3])
+
+      lb.select_healthy_node do |node|
+        # to handle random choice
+        if node == n1
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+        else
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+        end
+      end
+    end
+
     test 'raise an error if all node are unavialble' do
       lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
       lb.rebuild_weight_array([flexmock('node', :'standby?' => false, :'available?' => false, weight: 1)])
@@ -56,5 +94,13 @@ class LoadBalancerTest < Test::Unit::TestCase
         lb.select_healthy_node
       end
     end
+
+    test 'it regards weight=0 node as unavialble' do
+      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
+      lb.rebuild_weight_array([flexmock('node', :'standby?' => false, :'available?' => true, weight: 0)])
+      assert_raise(Fluent::Plugin::ForwardOutput::NoNodesAvailable) do
+        lb.select_healthy_node
+      end
+    end
   end
 end

data/test/plugin/test_in_gc_stat.rb

@@ -22,9 +22,14 @@ class GCStatInputTest < Test::Unit::TestCase
     assert_equal("t1", d.instance.tag)
   end
 
-  def test_emit
+  def setup_gc_stat
     stat = GC.stat
     stub(GC).stat { stat }
+    stat
+  end
+
+  def test_emit
+    stat = setup_gc_stat
 
     d = create_driver
     d.run(expect_emits: 2)
@@ -36,4 +41,22 @@ class GCStatInputTest < Test::Unit::TestCase
       assert(events[i][1].is_a?(Fluent::EventTime))
     }
   end
+
+  def test_emit_with_use_symbol_keys_false
+    stat = setup_gc_stat
+    result = {}
+    stat.each_pair { |k, v|
+      result[k.to_s] = v
+    }
+
+    d = create_driver(CONFIG + "use_symbol_keys false")
+    d.run(expect_emits: 2)
+
+    events = d.events
+    assert(events.length > 0)
+    events.each_index {|i|
+      assert_equal(result, events[i][2])
+      assert(events[i][1].is_a?(Fluent::EventTime))
+    }
+  end
 end

data/test/plugin/test_in_http.rb

@@ -116,6 +116,36 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal_event_time time, d.events[1][1]
   end
 
+  data('json' => ['json', :to_json],
+       'msgpack' => ['msgpack', :to_msgpack])
+  def test_default_with_time_format(data)
+    param, method_name = data
+    d = create_driver(CONFIG + %[
+      <parse>
+        keep_time_key
+        time_format %iso8601
+      </parse>
+    ])
+
+    time = event_time("2020-06-10T01:14:27+00:00")
+    events = [
+      ["tag1", time, {"a" => 1, "time" => '2020-06-10T01:14:27+00:00'}],
+      ["tag2", time, {"a" => 2, "time" => '2020-06-10T01:14:27+00:00'}],
+    ]
+    res_codes = []
+
+    d.run(expect_records: 2) do
+      events.each do |tag, t, record|
+        res = post("/#{tag}", {param => record.__send__(method_name)})
+        res_codes << res.code
+      end
+    end
+    assert_equal ["200", "200"], res_codes
+    assert_equal events, d.events
+    assert_equal_event_time time, d.events[0][1]
+    assert_equal_event_time time, d.events[1][1]
+  end
+
   def test_multi_json
     d = create_driver
     time = event_time("2011-01-02 13:14:15 UTC")
@@ -163,6 +193,33 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal_event_time time, d.events[1][1]
   end
 
+  data('json' => ['json', :to_json],
+       'msgpack' => ['msgpack', :to_msgpack])
+  def test_default_multi_with_time_format(data)
+    param, method_name = data
+    d = create_driver(CONFIG + %[
+      <parse>
+        keep_time_key
+        time_format %iso8601
+      </parse>
+    ])
+    time = event_time("2020-06-10T01:14:27+00:00")
+    events = [
+      ["tag1", time, {'a' => 1, 'time' => "2020-06-10T01:14:27+00:00"}],
+      ["tag1", time, {'a' => 2, 'time' => "2020-06-10T01:14:27+00:00"}],
+    ]
+    tag = "tag1"
+    res_codes = []
+    d.run(expect_records: 2, timeout: 5) do
+      res = post("/#{tag}", {param => events.map { |e| e[2] }.__send__(method_name)})
+      res_codes << res.code
+    end
+    assert_equal ["200"], res_codes
+    assert_equal events, d.events
+    assert_equal_event_time time, d.events[0][1]
+    assert_equal_event_time time, d.events[1][1]
+  end
+
   def test_multi_json_with_nonexistent_time_key
     d = create_driver(CONFIG + %[
       <parse>

data/test/plugin/{test_in_dummy.rb → test_in_sample.rb}

@@ -1,33 +1,33 @@
 require_relative '../helper'
 require 'fluent/test/driver/input'
-require 'fluent/plugin/in_dummy'
+require 'fluent/plugin/in_sample'
 require 'fileutils'
 
-class DummyTest < Test::Unit::TestCase
+class SampleTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
   end
 
   def create_driver(conf)
-    Fluent::Test::Driver::Input.new(Fluent::Plugin::DummyInput).configure(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::SampleInput).configure(conf)
   end
 
   sub_test_case 'configure' do
     test 'required parameters' do
       assert_raise_message("'tag' parameter is required") do
-        Fluent::Plugin::DummyInput.new.configure(config_element('ROOT',''))
+        Fluent::Plugin::SampleInput.new.configure(config_element('ROOT',''))
       end
     end
 
     test 'tag' do
       d = create_driver(%[
-        tag dummy
+        tag sample
       ])
-      assert_equal "dummy", d.instance.tag
+      assert_equal "sample", d.instance.tag
     end
 
     config = %[
-      tag dummy
+      tag sample
     ]
 
     test 'auto_increment_key' do
@@ -44,30 +44,30 @@ class DummyTest < Test::Unit::TestCase
       assert_equal 10, d.instance.rate
     end
 
-    test 'dummy' do
+    test 'sample' do
       # hash is okay
-      d = create_driver(config + %[dummy {"foo":"bar"}])
-      assert_equal [{"foo"=>"bar"}], d.instance.dummy
+      d = create_driver(config + %[sample {"foo":"bar"}])
+      assert_equal [{"foo"=>"bar"}], d.instance.sample
 
       # array of hash is okay
-      d = create_driver(config + %[dummy [{"foo":"bar"}]])
-      assert_equal [{"foo"=>"bar"}], d.instance.dummy
+      d = create_driver(config + %[sample [{"foo":"bar"}]])
+      assert_equal [{"foo"=>"bar"}], d.instance.sample
 
       assert_raise_message(/JSON::ParserError|got incomplete JSON/) do
-        create_driver(config + %[dummy "foo"])
+        create_driver(config + %[sample "foo"])
       end
 
       assert_raise_message(/is not a hash/) do
-        create_driver(config + %[dummy ["foo"]])
+        create_driver(config + %[sample ["foo"]])
       end
     end
   end
 
   sub_test_case "emit" do
     config = %[
-      tag dummy
+      tag sample
       rate 10
-      dummy {"foo":"bar"}
+      sample {"foo":"bar"}
     ]
 
     test 'simple' do
@@ -75,7 +75,7 @@ class DummyTest < Test::Unit::TestCase
       d.run(timeout: 0.5)
 
       d.events.each do |tag, time, record|
-        assert_equal("dummy", tag)
+        assert_equal("sample", tag)
         assert_equal({"foo"=>"bar"}, record)
         assert(time.is_a?(Fluent::EventTime))
       end
@@ -86,20 +86,20 @@ class DummyTest < Test::Unit::TestCase
       d.run(timeout: 0.5)
 
       d.events.each_with_index do |(tag, _time, record), i|
-        assert_equal("dummy", tag)
+        assert_equal("sample", tag)
         assert_equal({"foo"=>"bar", "id"=>i}, record)
       end
     end
   end
 
-  TEST_PLUGIN_STORAGE_PATH = File.join( File.dirname(File.dirname(__FILE__)), 'tmp', 'in_dummy', 'store' )
+  TEST_PLUGIN_STORAGE_PATH = File.join( File.dirname(File.dirname(__FILE__)), 'tmp', 'in_sample', 'store' )
   FileUtils.mkdir_p TEST_PLUGIN_STORAGE_PATH
 
-  sub_test_case 'when dummy plugin has storage which is not specified the path'  do
+  sub_test_case 'when sample plugin has storage which is not specified the path'  do
     config1 = {
-      'tag' => 'dummy',
+      'tag' => 'sample',
       'rate' => '0',
-      'dummy' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
+      'sample' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
       'auto_increment_key' => 'id',
     }
     conf1 = config_element('ROOT', '', config1, [])
@@ -135,7 +135,7 @@ class DummyTest < Test::Unit::TestCase
     end
   end
 
-  sub_test_case 'when dummy plugin has storage which is specified the path'  do
+  sub_test_case 'when sample plugin has storage which is specified the path'  do
     setup do
       FileUtils.rm_rf(TEST_PLUGIN_STORAGE_PATH)
       FileUtils.mkdir_p(File.join(TEST_PLUGIN_STORAGE_PATH, 'json'))
@@ -144,9 +144,9 @@ class DummyTest < Test::Unit::TestCase
 
     config2 = {
       '@id' => 'test-02',
-      'tag' => 'dummy',
+      'tag' => 'sample',
       'rate' => '0',
-      'dummy' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
+      'sample' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
       'auto_increment_key' => 'id',
     }
     conf2 = config_element('ROOT', '', config2, [