fluentd 1.11.0-x86-mingw32 → 1.11.5-x86-mingw32

data/lib/fluent/plugin/in_tail.rb

@@ -271,9 +271,9 @@ module Fluent::Plugin
                 end
               else
                 if is_file
-                  unless @ignore_list.include?(path)
+                  unless @ignore_list.include?(p)
                     log.warn "#{p} unreadable. It is excluded and would be examined next time."
-                    @ignore_list << path if @ignore_repeated_permission_error
+                    @ignore_list << p if @ignore_repeated_permission_error
                   end
                 end
                 false

data/lib/fluent/plugin/out_http.rb

@@ -21,6 +21,16 @@ require 'fluent/tls'
 require 'fluent/plugin/output'
 require 'fluent/plugin_helper/socket'
 
+# patch Net::HTTP to support extra_chain_cert which was added in Ruby feature #9758.
+# see: https://github.com/ruby/ruby/commit/31af0dafba6d3769d2a39617c0dddedb97883712
+unless Net::HTTP::SSL_IVNAMES.include?(:@extra_chain_cert)
+  class Net::HTTP
+    SSL_IVNAMES << :@extra_chain_cert
+    SSL_ATTRIBUTES << :extra_chain_cert
+    attr_accessor :extra_chain_cert
+  end
+end
+
 module Fluent::Plugin
   class HTTPOutput < Output
     Fluent::Plugin.register_output('http', self)
@@ -171,7 +181,15 @@ module Fluent::Plugin
         end
         if @tls_client_cert_path
           raise Fluent::ConfigError, "tls_client_cert_path is wrong: #{@tls_client_cert_path}" unless File.file?(@tls_client_cert_path)
-          opt[:cert] = OpenSSL::X509::Certificate.new(File.read(@tls_client_cert_path))
+
+          bundle = File.read(@tls_client_cert_path)
+          bundle_certs = bundle.scan(/-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/)
+          opt[:cert] = OpenSSL::X509::Certificate.new(bundle_certs[0])
+
+          intermediate_certs = bundle_certs[1..-1]
+          if intermediate_certs
+            opt[:extra_chain_cert] = intermediate_certs.map { |cert| OpenSSL::X509::Certificate.new(cert) }
+          end
         end
         if @tls_private_key_path
           raise Fluent::ConfigError, "tls_private_key_path is wrong: #{@tls_private_key_path}" unless File.file?(@tls_private_key_path)
@@ -215,7 +233,7 @@ module Fluent::Plugin
         req.basic_auth(@auth.username, @auth.password)
       end
       set_headers(req)
-      req.body = @json_array ? "[#{chunk.read.chop!}]" : chunk.read
+      req.body = @json_array ? "[#{chunk.read.chop}]" : chunk.read
       req
     end
 

data/lib/fluent/plugin/output.rb

@@ -340,6 +340,7 @@ module Fluent
           buffer_conf = conf.elements(name: 'buffer').first || Fluent::Config::Element.new('buffer', '', {}, [])
           @buffer = Plugin.new_buffer(buffer_type, parent: self)
           @buffer.configure(buffer_conf)
+          @buffer.enable_update_timekeys if @chunk_key_time
 
           @flush_at_shutdown = @buffer_config.flush_at_shutdown
           if @flush_at_shutdown.nil?
@@ -768,17 +769,19 @@ module Fluent
             end
           end
 
-          if rvalue =~ CHUNK_KEY_PLACEHOLDER_PATTERN
-            log.warn "chunk key placeholder '#{$1}' not replaced. template:#{str}"
-          end
-
-          rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
+          rvalue = rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
             if chunk_passed
               dump_unique_id_hex(chunk.unique_id)
             else
               log.warn "${chunk_id} is not allowed in this plugin. Pass Chunk instead of metadata in extract_placeholders's 2nd argument"
             end
           }
+
+          if rvalue =~ CHUNK_KEY_PLACEHOLDER_PATTERN
+            log.warn "chunk key placeholder '#{$1}' not replaced. template:#{str}"
+          end
+
+          rvalue
         end
       end
 

data/lib/fluent/plugin/parser_json.rb

@@ -71,8 +71,11 @@ module Fluent
       end
 
       def parse(text)
-        r = @load_proc.call(text)
-        time, record = convert_values(parse_time(r), r)
+        record = @load_proc.call(text)
+        time = parse_time(record)
+        if @execute_convert_values
+          time, record = convert_values(time, record)
+        end
         yield time, record
       rescue @error_class, EncodingError # EncodingError is for oj 3.x or later
         yield nil, nil

data/lib/fluent/plugin_helper/cert_option.rb

@@ -27,13 +27,9 @@ module Fluent
         cert, key, extra = cert_option_server_validate!(conf)
 
         ctx = OpenSSL::SSL::SSLContext.new
-        unless insecure
-          # inject OpenSSL::SSL::SSLContext::DEFAULT_PARAMS
-          # https://bugs.ruby-lang.org/issues/9424
-          ctx.set_params({})
-
-          ctx.ciphers = ciphers
-        end
+        # inject OpenSSL::SSL::SSLContext::DEFAULT_PARAMS
+        # https://bugs.ruby-lang.org/issues/9424
+        ctx.set_params({}) unless insecure
 
         if conf.client_cert_auth
           ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
@@ -56,6 +52,7 @@ module Fluent
         end
 
         Fluent::TLS.set_version_to_context(ctx, version, conf.min_version, conf.max_version)
+        ctx.ciphers = ciphers unless insecure
 
         ctx
       end
@@ -185,7 +182,7 @@ module Fluent
         list = []
         data.scan(pattern){|match| list << OpenSSL::X509::Certificate.new(match) }
         if list.length == 0
-          log.warn "cert_path does not contain a valid certificate"
+          raise Fluent::ConfigError, "cert_path does not contain a valid certificate"
         end
         list
       end

data/lib/fluent/plugin_helper/child_process.rb

@@ -259,6 +259,9 @@ module Fluent
 
         if !mode.include?(:stderr) && !mode.include?(:read_with_stderr)
           spawn_opts[:err] = IO::NULL if stderr == :discard
+          if !mode.include?(:read) && !mode.include?(:read_with_stderr)
+            spawn_opts[:out] = IO::NULL
+          end
           writeio, readio, wait_thread = *Open3.popen2(*spawn_args, spawn_opts)
         elsif mode.include?(:read_with_stderr)
           writeio, readio, wait_thread = *Open3.popen2e(*spawn_args, spawn_opts)
@@ -275,8 +278,6 @@ module Fluent
         if mode.include?(:read) || mode.include?(:read_with_stderr)
           readio.set_encoding(external_encoding, internal_encoding, **encoding_options)
           readio_in_use = true
-        else
-          readio.reopen(IO::NULL) if readio
         end
         if mode.include?(:stderr)
           stderrio.set_encoding(external_encoding, internal_encoding, **encoding_options)

data/lib/fluent/plugin_helper/inject.rb

@@ -76,7 +76,7 @@ module Fluent
           config_param :time_key, :string, default: nil
 
           # To avoid defining :time_type twice
-          config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :float
+          config_param :time_type, :enum, list: [:float, :unixtime, :unixtime_millis, :string], default: :float
 
           Fluent::TimeMixin::TIME_PARAMETERS.each do |name, type, opts|
             config_param(name, type, **opts)
@@ -132,6 +132,7 @@ module Fluent
           if @_inject_time_key
             @_inject_time_formatter = case @inject_config.time_type
                                       when :float then ->(time){ time.to_r.truncate(+6).to_f } # microsecond floating point value
+                                      when :unixtime_millis then ->(time) { time.to_f.floor(3) * 1000 }
                                       when :unixtime then ->(time){ time.to_i }
                                       else
                                         localtime = @inject_config.localtime && !@inject_config.utc

data/lib/fluent/plugin_helper/socket.rb

@@ -199,7 +199,7 @@ module Fluent
         list = []
         data.scan(pattern) { |match| list << OpenSSL::X509::Certificate.new(match) }
         if list.length == 0
-          log.warn "cert_path does not contain a valid certificate"
+          raise Fluent::ConfigError, "cert_path does not contain a valid certificate"
         end
         list
       end

data/lib/fluent/supervisor.rb

@@ -183,7 +183,6 @@ module Fluent
           until WaitForSingleObject(ev.handle, 0) == WAIT_OBJECT_0
             sleep 1
           end
-          kill_worker
           stop(true)
         ensure
           ev.close
@@ -302,6 +301,7 @@ module Fluent
       log_level = params['log_level']
       suppress_repeated_stacktrace = params['suppress_repeated_stacktrace']
       ignore_repeated_log_interval = params['ignore_repeated_log_interval']
+      ignore_same_log_interval = params['ignore_same_log_interval']
 
       log_path = params['log_path']
       chuser = params['chuser']
@@ -309,7 +309,8 @@ module Fluent
       log_rotate_age = params['log_rotate_age']
       log_rotate_size = params['log_rotate_size']
 
-      log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace, ignore_repeated_log_interval: ignore_repeated_log_interval}
+      log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace, ignore_repeated_log_interval: ignore_repeated_log_interval,
+                  ignore_same_log_interval: ignore_same_log_interval}
       logger_initializer = Supervisor::LoggerInitializer.new(
         log_path, log_level, chuser, chgroup, log_opts,
         log_rotate_age: log_rotate_age,
@@ -347,6 +348,7 @@ module Fluent
         chumask: 0,
         suppress_repeated_stacktrace: suppress_repeated_stacktrace,
         ignore_repeated_log_interval: ignore_repeated_log_interval,
+        ignore_same_log_interval: ignore_same_log_interval,
         daemonize: daemonize,
         rpc_endpoint: params['rpc_endpoint'],
         counter_server: params['counter_server'],
@@ -441,10 +443,11 @@ module Fluent
         self
       end
 
-      def apply_options(format: nil, time_format: nil, log_dir_perm: nil, ignore_repeated_log_interval: nil)
+      def apply_options(format: nil, time_format: nil, log_dir_perm: nil, ignore_repeated_log_interval: nil, ignore_same_log_interval: nil)
         $log.format = format if format
         $log.time_format = time_format if time_format
         $log.ignore_repeated_log_interval = ignore_repeated_log_interval if ignore_repeated_log_interval
+        $log.ignore_same_log_interval = ignore_same_log_interval if ignore_same_log_interval
 
         if @path && log_dir_perm
           File.chmod(log_dir_perm || 0755, File.dirname(@path))
@@ -511,7 +514,8 @@ module Fluent
       @cl_opt = opt
       @conf = nil
 
-      log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval]}
+      log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval],
+                  ignore_same_log_interval: opt[:ignore_same_log_interval]}
       @log = LoggerInitializer.new(
         @log_path, opt[:log_level], @chuser, @chgroup, log_opts,
         log_rotate_age: @log_rotate_age,
@@ -635,7 +639,8 @@ module Fluent
         format: @system_config.log.format,
         time_format: @system_config.log.time_format,
         log_dir_perm: @system_config.dir_permission,
-        ignore_repeated_log_interval: @system_config.ignore_repeated_log_interval
+        ignore_repeated_log_interval: @system_config.ignore_repeated_log_interval,
+        ignore_same_log_interval: @system_config.ignore_same_log_interval
       )
 
       $log.info :supervisor, 'parsing config file is succeeded', path: @config_path
@@ -804,7 +809,7 @@ module Fluent
             Fluent::Engine.reload_config(conf)
           end
         rescue => e
-          # it is guranteed that config file is valid by supervisor side. but it's not atomic becuase of using signals to commnicate between worker and super
+          # it is guaranteed that config file is valid by supervisor side. but it's not atomic because of using signals to commnicate between worker and super
           # So need this rescue code
           $log.error("failed to reload config: #{e}")
           next

data/lib/fluent/system_config.rb

@@ -24,7 +24,7 @@ module Fluent
     SYSTEM_CONFIG_PARAMETERS = [
       :workers, :root_dir, :log_level,
       :suppress_repeated_stacktrace, :emit_error_log_interval, :suppress_config_dump,
-      :log_event_verbose, :ignore_repeated_log_interval,
+      :log_event_verbose, :ignore_repeated_log_interval, :ignore_same_log_interval,
       :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
       :file_permission, :dir_permission, :counter_server, :counter_client,
       :strict_config_value, :enable_msgpack_time_support
@@ -35,6 +35,7 @@ module Fluent
     config_param :log_level, :enum, list: [:trace, :debug, :info, :warn, :error, :fatal], default: 'info'
     config_param :suppress_repeated_stacktrace, :bool, default: nil
     config_param :ignore_repeated_log_interval, :time, default: nil
+    config_param :ignore_same_log_interval, :time, default: nil
     config_param :emit_error_log_interval,      :time, default: nil
     config_param :suppress_config_dump, :bool, default: nil
     config_param :log_event_verbose,    :bool, default: nil

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.11.0'
+  VERSION = '1.11.5'
 
 end

data/test/command/test_binlog_reader.rb

@@ -82,6 +82,14 @@ class TestBaseCommand < ::Test::Unit::TestCase
 end
 
 class TestHead < TestBaseCommand
+  setup do
+    @default_newline = if Fluent.windows?
+                         "\r\n"
+                       else
+                         "\n"
+                       end
+  end
+
   sub_test_case 'initialize' do
     data(
       'file is not passed' => %w(),
@@ -138,7 +146,7 @@ class TestHead < TestBaseCommand
         create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
         head = BinlogReaderCommand::Head.new(argv)
         out = capture_stdout { head.call }
-        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}\n" * 5, out
+        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}#{@default_newline}" * 5, out
       end
     end
 
@@ -149,7 +157,7 @@ class TestHead < TestBaseCommand
         create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
         head = BinlogReaderCommand::Head.new(argv)
         out = capture_stdout { head.call }
-        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}\n", out
+        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}#{@default_newline}", out
       end
     end
 
@@ -169,7 +177,7 @@ class TestHead < TestBaseCommand
         create_message_packed_file(@file_name, [event_time(@t).to_i], [@record])
         head = BinlogReaderCommand::Head.new(argv)
         out = capture_stdout { head.call }
-        assert_equal "#{Yajl.dump(@record)}\n", out
+        assert_equal "#{Yajl.dump(@record)}#{@default_newline}", out
       end
     end
 
@@ -198,6 +206,14 @@ class TestHead < TestBaseCommand
 end
 
 class TestCat < TestBaseCommand
+  setup do
+    @default_newline = if Fluent.windows?
+                         "\r\n"
+                       else
+                         "\n"
+                       end
+  end
+
   sub_test_case 'initialize' do
     data(
       'file is not passed' => [],
@@ -254,7 +270,7 @@ class TestCat < TestBaseCommand
         create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
         head = BinlogReaderCommand::Cat.new(argv)
         out = capture_stdout { head.call }
-        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}\n" * 6, out
+        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}#{@default_newline}" * 6, out
       end
     end
 
@@ -265,7 +281,7 @@ class TestCat < TestBaseCommand
         create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
         head = BinlogReaderCommand::Cat.new(argv)
         out = capture_stdout { head.call }
-        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}\n", out
+        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}#{@default_newline}", out
       end
     end
 
@@ -276,7 +292,7 @@ class TestCat < TestBaseCommand
         create_message_packed_file(@file_name, [event_time(@t).to_i], [@record])
         head = BinlogReaderCommand::Cat.new(argv)
         out = capture_stdout { head.call }
-        assert_equal "#{Yajl.dump(@record)}\n", out
+        assert_equal "#{Yajl.dump(@record)}#{@default_newline}", out
       end
     end
 

data/test/plugin/test_buffer.rb

@@ -543,6 +543,8 @@ class BufferTest < Test::Unit::TestCase
       assert_equal [@dm0,@dm1,@dm1], @p.queue.map(&:metadata)
       assert_equal [@dm2,@dm3,m], @p.stage.keys
 
+      @p.update_timekeys
+
       assert @p.timekeys.include?(timekey)
     end
 
@@ -675,6 +677,8 @@ class BufferTest < Test::Unit::TestCase
       assert_equal [@dm2,@dm3,m], @p.stage.keys
       assert_equal 1, @p.stage[m].append_count
 
+      @p.update_timekeys
+
       assert @p.timekeys.include?(timekey)
     end
 

data/test/plugin/test_filter_stdout.rb

@@ -12,6 +12,11 @@ class StdoutFilterTest < Test::Unit::TestCase
     @old_tz = ENV["TZ"]
     ENV["TZ"] = "UTC"
     Timecop.freeze
+    @default_newline = if Fluent.windows?
+                         "\r\n"
+                       else
+                         "\n"
+                       end
   end
 
   def teardown
@@ -106,7 +111,7 @@ class StdoutFilterTest < Test::Unit::TestCase
     def test_format_json
       d = create_driver(CONFIG + config_element("", "", { "format" => "json" }))
       out = capture_log(d) { filter(d, event_time, {'test' => 'test'}) }
-      assert_equal "{\"test\":\"test\"}\n", out
+      assert_equal "{\"test\":\"test\"}#{@default_newline}", out
     end
   end
 

data/test/plugin/test_formatter_hash.rb

@@ -19,11 +19,14 @@ class HashFormatterTest < ::Test::Unit::TestCase
     {'message' => 'awesome', 'greeting' => 'hello'}
   end
 
-  def test_format
-    d = create_driver({})
+  data("newline (LF)" => ["lf", "\n"],
+       "newline (CRLF)" => ["crlf", "\r\n"])
+  def test_format(data)
+    newline_conf, newline = data
+    d = create_driver({"newline" => newline_conf})
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal(%Q!{"message"=>"awesome", "greeting"=>"hello"}\n!, formatted.encode(Encoding::UTF_8))
+    assert_equal(%Q!{"message"=>"awesome", "greeting"=>"hello"}#{newline}!, formatted.encode(Encoding::UTF_8))
   end
 
   def test_format_without_newline

data/test/plugin/test_formatter_json.rb

@@ -7,6 +7,11 @@ class JsonFormatterTest < ::Test::Unit::TestCase
 
   def setup
     @time = event_time
+    @default_newline = if Fluent.windows?
+                         "\r\n"
+                       else
+                         "\n"
+                       end
   end
 
   def create_driver(conf = "")
@@ -25,12 +30,17 @@ class JsonFormatterTest < ::Test::Unit::TestCase
     {:message => :awesome}
   end
 
-  data('oj' => 'oj', 'yajl' => 'yajl')
+  data('oj with LF' => ['oj', "lf", "\n"],
+       'oj with CRLF' => ['oj', "crlf", "\r\n"],
+       'yajl with LF' => ['yajl', "lf", "\n"],
+       'yajl with CRLF' => ['yajl', "crlf", "\r\n"]
+      )
   def test_format(data)
-    d = create_driver('json_parser' => data)
+    parser, newline_conf, newline = data
+    d = create_driver('json_parser' => parser, 'newline' => newline_conf)
     formatted = d.instance.format(tag, @time, record)
 
-    assert_equal("#{JSON.generate(record)}\n", formatted)
+    assert_equal("#{JSON.generate(record)}#{newline}", formatted)
   end
 
   data('oj' => 'oj', 'yajl' => 'yajl')
@@ -46,6 +56,6 @@ class JsonFormatterTest < ::Test::Unit::TestCase
     d = create_driver('json_parser' => data)
     formatted = d.instance.format(tag, @time, symbolic_record)
 
-    assert_equal("#{JSON.generate(record)}\n", formatted)
+    assert_equal("#{JSON.generate(record)}#{@default_newline}", formatted)
   end
 end