fluentd 1.10.4-x86-mingw32 → 1.11.4-x86-mingw32

data/lib/fluent/plugin_helper/cert_option.rb

@@ -27,13 +27,9 @@ module Fluent
         cert, key, extra = cert_option_server_validate!(conf)
 
         ctx = OpenSSL::SSL::SSLContext.new
-        unless insecure
-          # inject OpenSSL::SSL::SSLContext::DEFAULT_PARAMS
-          # https://bugs.ruby-lang.org/issues/9424
-          ctx.set_params({})
-
-          ctx.ciphers = ciphers
-        end
+        # inject OpenSSL::SSL::SSLContext::DEFAULT_PARAMS
+        # https://bugs.ruby-lang.org/issues/9424
+        ctx.set_params({}) unless insecure
 
         if conf.client_cert_auth
           ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
@@ -56,6 +52,7 @@ module Fluent
         end
 
         Fluent::TLS.set_version_to_context(ctx, version, conf.min_version, conf.max_version)
+        ctx.ciphers = ciphers unless insecure
 
         ctx
       end
@@ -185,7 +182,7 @@ module Fluent
         list = []
         data.scan(pattern){|match| list << OpenSSL::X509::Certificate.new(match) }
         if list.length == 0
-          log.warn "cert_path does not contain a valid certificate"
+          raise Fluent::ConfigError, "cert_path does not contain a valid certificate"
         end
         list
       end

data/lib/fluent/plugin_helper/child_process.rb

@@ -259,6 +259,9 @@ module Fluent
 
         if !mode.include?(:stderr) && !mode.include?(:read_with_stderr)
           spawn_opts[:err] = IO::NULL if stderr == :discard
+          if !mode.include?(:read) && !mode.include?(:read_with_stderr)
+            spawn_opts[:out] = IO::NULL
+          end
           writeio, readio, wait_thread = *Open3.popen2(*spawn_args, spawn_opts)
         elsif mode.include?(:read_with_stderr)
           writeio, readio, wait_thread = *Open3.popen2e(*spawn_args, spawn_opts)
@@ -275,8 +278,6 @@ module Fluent
         if mode.include?(:read) || mode.include?(:read_with_stderr)
           readio.set_encoding(external_encoding, internal_encoding, **encoding_options)
           readio_in_use = true
-        else
-          readio.reopen(IO::NULL) if readio
         end
         if mode.include?(:stderr)
           stderrio.set_encoding(external_encoding, internal_encoding, **encoding_options)

data/lib/fluent/plugin_helper/inject.rb

@@ -76,7 +76,7 @@ module Fluent
           config_param :time_key, :string, default: nil
 
           # To avoid defining :time_type twice
-          config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :float
+          config_param :time_type, :enum, list: [:float, :unixtime, :unixtime_millis, :string], default: :float
 
           Fluent::TimeMixin::TIME_PARAMETERS.each do |name, type, opts|
             config_param(name, type, **opts)
@@ -132,6 +132,7 @@ module Fluent
           if @_inject_time_key
             @_inject_time_formatter = case @inject_config.time_type
                                       when :float then ->(time){ time.to_r.truncate(+6).to_f } # microsecond floating point value
+                                      when :unixtime_millis then ->(time) { time.to_f.floor(3) * 1000 }
                                       when :unixtime then ->(time){ time.to_i }
                                       else
                                         localtime = @inject_config.localtime && !@inject_config.utc

data/lib/fluent/plugin_helper/socket.rb

@@ -199,7 +199,7 @@ module Fluent
         list = []
         data.scan(pattern) { |match| list << OpenSSL::X509::Certificate.new(match) }
         if list.length == 0
-          log.warn "cert_path does not contain a valid certificate"
+          raise Fluent::ConfigError, "cert_path does not contain a valid certificate"
         end
         list
       end

data/lib/fluent/supervisor.rb

@@ -302,6 +302,7 @@ module Fluent
       log_level = params['log_level']
       suppress_repeated_stacktrace = params['suppress_repeated_stacktrace']
       ignore_repeated_log_interval = params['ignore_repeated_log_interval']
+      ignore_same_log_interval = params['ignore_same_log_interval']
 
       log_path = params['log_path']
       chuser = params['chuser']
@@ -309,7 +310,8 @@ module Fluent
       log_rotate_age = params['log_rotate_age']
       log_rotate_size = params['log_rotate_size']
 
-      log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace, ignore_repeated_log_interval: ignore_repeated_log_interval}
+      log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace, ignore_repeated_log_interval: ignore_repeated_log_interval,
+                  ignore_same_log_interval: ignore_same_log_interval}
       logger_initializer = Supervisor::LoggerInitializer.new(
         log_path, log_level, chuser, chgroup, log_opts,
         log_rotate_age: log_rotate_age,
@@ -347,6 +349,7 @@ module Fluent
         chumask: 0,
         suppress_repeated_stacktrace: suppress_repeated_stacktrace,
         ignore_repeated_log_interval: ignore_repeated_log_interval,
+        ignore_same_log_interval: ignore_same_log_interval,
         daemonize: daemonize,
         rpc_endpoint: params['rpc_endpoint'],
         counter_server: params['counter_server'],
@@ -441,10 +444,11 @@ module Fluent
         self
       end
 
-      def apply_options(format: nil, time_format: nil, log_dir_perm: nil, ignore_repeated_log_interval: nil)
+      def apply_options(format: nil, time_format: nil, log_dir_perm: nil, ignore_repeated_log_interval: nil, ignore_same_log_interval: nil)
         $log.format = format if format
         $log.time_format = time_format if time_format
         $log.ignore_repeated_log_interval = ignore_repeated_log_interval if ignore_repeated_log_interval
+        $log.ignore_same_log_interval = ignore_same_log_interval if ignore_same_log_interval
 
         if @path && log_dir_perm
           File.chmod(log_dir_perm || 0755, File.dirname(@path))
@@ -511,7 +515,8 @@ module Fluent
       @cl_opt = opt
       @conf = nil
 
-      log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval]}
+      log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval],
+                  ignore_same_log_interval: opt[:ignore_same_log_interval]}
       @log = LoggerInitializer.new(
         @log_path, opt[:log_level], @chuser, @chgroup, log_opts,
         log_rotate_age: @log_rotate_age,
@@ -635,7 +640,8 @@ module Fluent
         format: @system_config.log.format,
         time_format: @system_config.log.time_format,
         log_dir_perm: @system_config.dir_permission,
-        ignore_repeated_log_interval: @system_config.ignore_repeated_log_interval
+        ignore_repeated_log_interval: @system_config.ignore_repeated_log_interval,
+        ignore_same_log_interval: @system_config.ignore_same_log_interval
       )
 
       $log.info :supervisor, 'parsing config file is succeeded', path: @config_path
@@ -804,7 +810,7 @@ module Fluent
             Fluent::Engine.reload_config(conf)
           end
         rescue => e
-          # it is guranteed that config file is valid by supervisor side. but it's not atomic becuase of using signals to commnicate between worker and super
+          # it is guaranteed that config file is valid by supervisor side. but it's not atomic because of using signals to commnicate between worker and super
           # So need this rescue code
           $log.error("failed to reload config: #{e}")
           next

data/lib/fluent/system_config.rb

@@ -24,7 +24,7 @@ module Fluent
     SYSTEM_CONFIG_PARAMETERS = [
       :workers, :root_dir, :log_level,
       :suppress_repeated_stacktrace, :emit_error_log_interval, :suppress_config_dump,
-      :log_event_verbose, :ignore_repeated_log_interval,
+      :log_event_verbose, :ignore_repeated_log_interval, :ignore_same_log_interval,
       :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
       :file_permission, :dir_permission, :counter_server, :counter_client,
       :strict_config_value, :enable_msgpack_time_support
@@ -35,6 +35,7 @@ module Fluent
     config_param :log_level, :enum, list: [:trace, :debug, :info, :warn, :error, :fatal], default: 'info'
     config_param :suppress_repeated_stacktrace, :bool, default: nil
     config_param :ignore_repeated_log_interval, :time, default: nil
+    config_param :ignore_same_log_interval, :time, default: nil
     config_param :emit_error_log_interval,      :time, default: nil
     config_param :suppress_config_dump, :bool, default: nil
     config_param :log_event_verbose,    :bool, default: nil

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.10.4'
+  VERSION = '1.11.4'
 
 end

data/test/plugin/test_buffer.rb

@@ -543,6 +543,8 @@ class BufferTest < Test::Unit::TestCase
       assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
       assert_equal [@dm2,@dm3,m], @p.stage.keys
 
+      @p.update_timekeys
+
       assert @p.timekeys.include?(timekey)
     end
 
@@ -675,6 +677,8 @@ class BufferTest < Test::Unit::TestCase
       assert_equal [@dm2,@dm3,m], @p.stage.keys
       assert_equal 1, @p.stage[m].append_count
 
+      @p.update_timekeys
+
       assert @p.timekeys.include?(timekey)
     end
 

data/test/plugin/test_in_exec.rb

@@ -240,4 +240,22 @@ EOC
       assert_equal [tag, time, record], event
     }
   end
+
+  test 'emit error message with read_with_stderr' do
+    d = create_driver %[
+      tag test
+      command ruby #{File.join(File.dirname(SCRIPT_PATH), 'foo_bar_baz_no_existence.rb')}
+      connect_mode read_with_stderr
+      <parse>
+        @type none
+      </parse>
+    ]
+    d.run(expect_records: 1, timeout: 10)
+
+    assert{ d.events.length > 0 }
+    d.events.each do |event|
+      assert_equal 'test', event[0]
+      assert_match /LoadError/, event[2]['message']
+    end
+  end
 end

data/test/plugin/test_in_gc_stat.rb

@@ -22,9 +22,14 @@ class GCStatInputTest < Test::Unit::TestCase
     assert_equal("t1", d.instance.tag)
   end
 
-  def test_emit
+  def setup_gc_stat
     stat = GC.stat
     stub(GC).stat { stat }
+    stat
+  end
+
+  def test_emit
+    stat = setup_gc_stat
 
     d = create_driver
     d.run(expect_emits: 2)
@@ -36,4 +41,22 @@ class GCStatInputTest < Test::Unit::TestCase
       assert(events[i][1].is_a?(Fluent::EventTime))
     }
   end
+
+  def test_emit_with_use_symbol_keys_false
+    stat = setup_gc_stat
+    result = {}
+    stat.each_pair { |k, v|
+      result[k.to_s] = v
+    }
+
+    d = create_driver(CONFIG + "use_symbol_keys false")
+    d.run(expect_emits: 2)
+
+    events = d.events
+    assert(events.length > 0)
+    events.each_index {|i|
+      assert_equal(result, events[i][2])
+      assert(events[i][1].is_a?(Fluent::EventTime))
+    }
+  end
 end

data/test/plugin/test_in_http.rb

@@ -116,6 +116,36 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal_event_time time, d.events[1][1]
   end
 
+  data('json' => ['json', :to_json],
+       'msgpack' => ['msgpack', :to_msgpack])
+  def test_default_with_time_format(data)
+    param, method_name = data
+    d = create_driver(CONFIG + %[
+      <parse>
+        keep_time_key
+        time_format %iso8601
+      </parse>
+    ])
+
+    time = event_time("2020-06-10T01:14:27+00:00")
+    events = [
+      ["tag1", time, {"a" => 1, "time" => '2020-06-10T01:14:27+00:00'}],
+      ["tag2", time, {"a" => 2, "time" => '2020-06-10T01:14:27+00:00'}],
+    ]
+    res_codes = []
+
+    d.run(expect_records: 2) do
+      events.each do |tag, t, record|
+        res = post("/#{tag}", {param => record.__send__(method_name)})
+        res_codes << res.code
+      end
+    end
+    assert_equal ["200", "200"], res_codes
+    assert_equal events, d.events
+    assert_equal_event_time time, d.events[0][1]
+    assert_equal_event_time time, d.events[1][1]
+  end
+
   def test_multi_json
     d = create_driver
     time = event_time("2011-01-02 13:14:15 UTC")
@@ -163,6 +193,33 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal_event_time time, d.events[1][1]
   end
 
+  data('json' => ['json', :to_json],
+       'msgpack' => ['msgpack', :to_msgpack])
+  def test_default_multi_with_time_format(data)
+    param, method_name = data
+    d = create_driver(CONFIG + %[
+      <parse>
+        keep_time_key
+        time_format %iso8601
+      </parse>
+    ])
+    time = event_time("2020-06-10T01:14:27+00:00")
+    events = [
+      ["tag1", time, {'a' => 1, 'time' => "2020-06-10T01:14:27+00:00"}],
+      ["tag1", time, {'a' => 2, 'time' => "2020-06-10T01:14:27+00:00"}],
+    ]
+    tag = "tag1"
+    res_codes = []
+    d.run(expect_records: 2, timeout: 5) do
+      res = post("/#{tag}", {param => events.map { |e| e[2] }.__send__(method_name)})
+      res_codes << res.code
+    end
+    assert_equal ["200"], res_codes
+    assert_equal events, d.events
+    assert_equal_event_time time, d.events[0][1]
+    assert_equal_event_time time, d.events[1][1]
+  end
+
   def test_multi_json_with_nonexistent_time_key
     d = create_driver(CONFIG + %[
       <parse>

data/test/plugin/{test_in_dummy.rb → test_in_sample.rb}

@@ -1,33 +1,33 @@
 require_relative '../helper'
 require 'fluent/test/driver/input'
-require 'fluent/plugin/in_dummy'
+require 'fluent/plugin/in_sample'
 require 'fileutils'
 
-class DummyTest < Test::Unit::TestCase
+class SampleTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
   end
 
   def create_driver(conf)
-    Fluent::Test::Driver::Input.new(Fluent::Plugin::DummyInput).configure(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::SampleInput).configure(conf)
   end
 
   sub_test_case 'configure' do
     test 'required parameters' do
       assert_raise_message("'tag' parameter is required") do
-        Fluent::Plugin::DummyInput.new.configure(config_element('ROOT',''))
+        Fluent::Plugin::SampleInput.new.configure(config_element('ROOT',''))
       end
     end
 
     test 'tag' do
       d = create_driver(%[
-        tag dummy
+        tag sample
       ])
-      assert_equal "dummy", d.instance.tag
+      assert_equal "sample", d.instance.tag
     end
 
     config = %[
-      tag dummy
+      tag sample
     ]
 
     test 'auto_increment_key' do
@@ -44,30 +44,30 @@ class DummyTest < Test::Unit::TestCase
       assert_equal 10, d.instance.rate
     end
 
-    test 'dummy' do
+    test 'sample' do
       # hash is okay
-      d = create_driver(config + %[dummy {"foo":"bar"}])
-      assert_equal [{"foo"=>"bar"}], d.instance.dummy
+      d = create_driver(config + %[sample {"foo":"bar"}])
+      assert_equal [{"foo"=>"bar"}], d.instance.sample
 
       # array of hash is okay
-      d = create_driver(config + %[dummy [{"foo":"bar"}]])
-      assert_equal [{"foo"=>"bar"}], d.instance.dummy
+      d = create_driver(config + %[sample [{"foo":"bar"}]])
+      assert_equal [{"foo"=>"bar"}], d.instance.sample
 
       assert_raise_message(/JSON::ParserError|got incomplete JSON/) do
-        create_driver(config + %[dummy "foo"])
+        create_driver(config + %[sample "foo"])
       end
 
       assert_raise_message(/is not a hash/) do
-        create_driver(config + %[dummy ["foo"]])
+        create_driver(config + %[sample ["foo"]])
       end
     end
   end
 
   sub_test_case "emit" do
     config = %[
-      tag dummy
+      tag sample
       rate 10
-      dummy {"foo":"bar"}
+      sample {"foo":"bar"}
     ]
 
     test 'simple' do
@@ -75,7 +75,7 @@ class DummyTest < Test::Unit::TestCase
       d.run(timeout: 0.5)
 
       d.events.each do |tag, time, record|
-        assert_equal("dummy", tag)
+        assert_equal("sample", tag)
         assert_equal({"foo"=>"bar"}, record)
         assert(time.is_a?(Fluent::EventTime))
       end
@@ -86,20 +86,20 @@ class DummyTest < Test::Unit::TestCase
       d.run(timeout: 0.5)
 
       d.events.each_with_index do |(tag, _time, record), i|
-        assert_equal("dummy", tag)
+        assert_equal("sample", tag)
         assert_equal({"foo"=>"bar", "id"=>i}, record)
       end
     end
   end
 
-  TEST_PLUGIN_STORAGE_PATH = File.join( File.dirname(File.dirname(__FILE__)), 'tmp', 'in_dummy', 'store' )
+  TEST_PLUGIN_STORAGE_PATH = File.join( File.dirname(File.dirname(__FILE__)), 'tmp', 'in_sample', 'store' )
   FileUtils.mkdir_p TEST_PLUGIN_STORAGE_PATH
 
-  sub_test_case 'when dummy plugin has storage which is not specified the path'  do
+  sub_test_case 'when sample plugin has storage which is not specified the path'  do
     config1 = {
-      'tag' => 'dummy',
+      'tag' => 'sample',
       'rate' => '0',
-      'dummy' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
+      'sample' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
       'auto_increment_key' => 'id',
     }
     conf1 = config_element('ROOT', '', config1, [])
@@ -135,7 +135,7 @@ class DummyTest < Test::Unit::TestCase
     end
   end
 
-  sub_test_case 'when dummy plugin has storage which is specified the path'  do
+  sub_test_case 'when sample plugin has storage which is specified the path'  do
     setup do
       FileUtils.rm_rf(TEST_PLUGIN_STORAGE_PATH)
       FileUtils.mkdir_p(File.join(TEST_PLUGIN_STORAGE_PATH, 'json'))
@@ -144,9 +144,9 @@ class DummyTest < Test::Unit::TestCase
 
     config2 = {
       '@id' => 'test-02',
-      'tag' => 'dummy',
+      'tag' => 'sample',
       'rate' => '0',
-      'dummy' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
+      'sample' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
       'auto_increment_key' => 'id',
     }
     conf2 = config_element('ROOT', '', config2, [