fluentd 1.11.2-x64-mingw32 → 1.12.1-x64-mingw32

data/lib/fluent/plugin/in_tcp.rb

@@ -98,6 +98,7 @@ module Fluent::Plugin
     def start
       super
 
+      log.info "listening tcp socket", bind: @bind, port: @port
       del_size = @delimiter.length
       if @_extract_enabled && @_extract_tag_key
         server_create(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key) do |data, conn|

data/lib/fluent/plugin/out_http.rb

@@ -21,6 +21,16 @@ require 'fluent/tls'
 require 'fluent/plugin/output'
 require 'fluent/plugin_helper/socket'
 
+# patch Net::HTTP to support extra_chain_cert which was added in Ruby feature #9758.
+# see: https://github.com/ruby/ruby/commit/31af0dafba6d3769d2a39617c0dddedb97883712
+unless Net::HTTP::SSL_IVNAMES.include?(:@extra_chain_cert)
+  class Net::HTTP
+    SSL_IVNAMES << :@extra_chain_cert
+    SSL_ATTRIBUTES << :extra_chain_cert
+    attr_accessor :extra_chain_cert
+  end
+end
+
 module Fluent::Plugin
   class HTTPOutput < Output
     Fluent::Plugin.register_output('http', self)
@@ -41,6 +51,8 @@ module Fluent::Plugin
     config_param :json_array, :bool, default: false
     desc 'Additional headers for HTTP request'
     config_param :headers, :hash, default: nil
+    desc 'Additional placeholder based headers for HTTP request'
+    config_param :headers_from_placeholders, :hash, default: nil
 
     desc 'The connection open timeout in seconds'
     config_param :open_timeout, :integer, default: nil
@@ -171,7 +183,15 @@ module Fluent::Plugin
         end
         if @tls_client_cert_path
           raise Fluent::ConfigError, "tls_client_cert_path is wrong: #{@tls_client_cert_path}" unless File.file?(@tls_client_cert_path)
-          opt[:cert] = OpenSSL::X509::Certificate.new(File.read(@tls_client_cert_path))
+
+          bundle = File.read(@tls_client_cert_path)
+          bundle_certs = bundle.scan(/-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/)
+          opt[:cert] = OpenSSL::X509::Certificate.new(bundle_certs[0])
+
+          intermediate_certs = bundle_certs[1..-1]
+          if intermediate_certs
+            opt[:extra_chain_cert] = intermediate_certs.map { |cert| OpenSSL::X509::Certificate.new(cert) }
+          end
         end
         if @tls_private_key_path
           raise Fluent::ConfigError, "tls_private_key_path is wrong: #{@tls_private_key_path}" unless File.file?(@tls_private_key_path)
@@ -195,12 +215,17 @@ module Fluent::Plugin
       URI.parse(endpoint)
     end
 
-    def set_headers(req)
+    def set_headers(req, chunk)
       if @headers
         @headers.each do |k, v|
           req[k] = v
         end
       end
+      if @headers_from_placeholders
+        @headers_from_placeholders.each do |k, v|
+          req[k] = extract_placeholders(v, chunk)
+        end
+      end
       req['Content-Type'] = @content_type
     end
 
@@ -214,8 +239,8 @@ module Fluent::Plugin
       if @auth
         req.basic_auth(@auth.username, @auth.password)
       end
-      set_headers(req)
-      req.body = @json_array ? "[#{chunk.read.chop!}]" : chunk.read
+      set_headers(req, chunk)
+      req.body = @json_array ? "[#{chunk.read.chop}]" : chunk.read
       req
     end
 

data/lib/fluent/plugin/output.rb

@@ -340,6 +340,7 @@ module Fluent
           buffer_conf = conf.elements(name: 'buffer').first || Fluent::Config::Element.new('buffer', '', {}, [])
           @buffer = Plugin.new_buffer(buffer_type, parent: self)
           @buffer.configure(buffer_conf)
+          @buffer.enable_update_timekeys if @chunk_key_time
 
           @flush_at_shutdown = @buffer_config.flush_at_shutdown
           if @flush_at_shutdown.nil?
@@ -768,17 +769,19 @@ module Fluent
             end
           end
 
-          if rvalue =~ CHUNK_KEY_PLACEHOLDER_PATTERN
-            log.warn "chunk key placeholder '#{$1}' not replaced. template:#{str}"
-          end
-
-          rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
+          rvalue = rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
             if chunk_passed
               dump_unique_id_hex(chunk.unique_id)
             else
               log.warn "${chunk_id} is not allowed in this plugin. Pass Chunk instead of metadata in extract_placeholders's 2nd argument"
             end
           }
+
+          if rvalue =~ CHUNK_KEY_PLACEHOLDER_PATTERN
+            log.warn "chunk key placeholder '#{$1}' not replaced. template:#{str}"
+          end
+
+          rvalue
         end
       end
 
@@ -1249,7 +1252,13 @@ module Fluent
             log.debug "buffer queue cleared"
             @retry = nil
           else
-            @retry.step
+            # Ensure that the current time is greater than or equal to @retry.next_time to avoid the situation when
+            # @retry.step is called almost as many times as the number of flush threads in a short time.
+            if Time.now >= @retry.next_time
+              @retry.step
+            else
+              @retry.recalc_next_time # to prevent all flush threads from retrying at the same time
+            end
             if error
               if using_secondary
                 msg = "failed to flush the buffer with secondary output."

data/lib/fluent/plugin/parser_json.rb

@@ -71,8 +71,11 @@ module Fluent
       end
 
       def parse(text)
-        r = @load_proc.call(text)
-        time, record = convert_values(parse_time(r), r)
+        record = @load_proc.call(text)
+        time = parse_time(record)
+        if @execute_convert_values
+          time, record = convert_values(time, record)
+        end
         yield time, record
       rescue @error_class, EncodingError # EncodingError is for oj 3.x or later
         yield nil, nil

data/lib/fluent/plugin_helper/http_server/compat/server.rb

@@ -81,7 +81,7 @@ module Fluent
 
           def build_handler
             @methods.group_by(&:first).each do |(path, rest)|
-              klass = Fluent::PluginHelper::HttpServer::Compat::WebrickHandler.build(Hash[rest.map { |e| [e[1], e[2]] }])
+              klass = Fluent::PluginHelper::HttpServer::Compat::WebrickHandler.build(**Hash[rest.map { |e| [e[1], e[2]] }])
               @server.mount(path, klass)
             end
           end

data/lib/fluent/plugin_helper/inject.rb

@@ -76,7 +76,7 @@ module Fluent
           config_param :time_key, :string, default: nil
 
           # To avoid defining :time_type twice
-          config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :float
+          config_param :time_type, :enum, list: [:float, :unixtime, :unixtime_millis, :unixtime_micros, :unixtime_nanos, :string], default: :float
 
           Fluent::TimeMixin::TIME_PARAMETERS.each do |name, type, opts|
             config_param(name, type, **opts)
@@ -132,6 +132,9 @@ module Fluent
           if @_inject_time_key
             @_inject_time_formatter = case @inject_config.time_type
                                       when :float then ->(time){ time.to_r.truncate(+6).to_f } # microsecond floating point value
+                                      when :unixtime_millis then ->(time) { time.respond_to?(:nsec) ? time.to_i * 1_000 + time.nsec / 1_000_000 : (time * 1_000).floor }
+                                      when :unixtime_micros then ->(time) { time.respond_to?(:nsec) ? time.to_i * 1_000_000 + time.nsec / 1_000 : (time * 1_000_000).floor }
+                                      when :unixtime_nanos then ->(time) { time.respond_to?(:nsec) ? time.to_i * 1_000_000_000 + time.nsec : (time * 1_000_000_000).floor }
                                       when :unixtime then ->(time){ time.to_i }
                                       else
                                         localtime = @inject_config.localtime && !@inject_config.utc

data/lib/fluent/plugin_helper/retry_state.rb

@@ -127,6 +127,10 @@ module Fluent
           nil
         end
 
+        def recalc_next_time
+          @next_time = calc_next_time
+        end
+
         def limit?
           if @forever
             false

data/lib/fluent/supervisor.rb

@@ -45,6 +45,7 @@ module Fluent
   module ServerModule
     def before_run
       @fluentd_conf = config[:fluentd_conf]
+      @rpc_endpoint = nil
       @rpc_server = nil
       @counter = nil
 
@@ -53,23 +54,28 @@ module Fluent
         @enable_get_dump = config[:enable_get_dump]
         run_rpc_server
       end
-      install_supervisor_signal_handlers
 
-      if config[:signame]
-        @signame = config[:signame]
+      if Fluent.windows?
         install_windows_event_handler
+      else
+        install_supervisor_signal_handlers
       end
 
       if counter = config[:counter_server]
         run_counter_server(counter)
       end
 
-      socket_manager_path = ServerEngine::SocketManager::Server.generate_path
-      ServerEngine::SocketManager::Server.open(socket_manager_path)
-      ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+      if config[:disable_shared_socket]
+        $log.info "shared socket for multiple workers is disabled"
+      else
+        socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+        ServerEngine::SocketManager::Server.open(socket_manager_path)
+        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+      end
     end
 
     def after_run
+      stop_windows_event_thread if Fluent.windows?
       stop_rpc_server if @rpc_endpoint
       stop_counter_server if @counter
       Fluent::Supervisor.cleanup_resources
@@ -92,7 +98,8 @@ module Fluent
       @rpc_server.mount_proc('/api/processes.flushBuffersAndKillWorkers') { |req, res|
         $log.debug "fluentd RPC got /api/processes.flushBuffersAndKillWorkers request"
         if Fluent.windows?
-          $log.warn "operation 'flushBuffersAndKillWorkers' is not supported on Windows now."
+          supervisor_sigusr1_handler
+          stop(true)
         else
           Process.kill :USR1, $$
           Process.kill :TERM, $$
@@ -101,7 +108,9 @@ module Fluent
       }
       @rpc_server.mount_proc('/api/plugins.flushBuffers') { |req, res|
         $log.debug "fluentd RPC got /api/plugins.flushBuffers request"
-        unless Fluent.windows?
+        if Fluent.windows?
+          supervisor_sigusr1_handler
+        else
           Process.kill :USR1, $$
         end
         nil
@@ -125,7 +134,9 @@ module Fluent
 
       @rpc_server.mount_proc('/api/config.gracefulReload') { |req, res|
         $log.debug "fluentd RPC got /api/config.gracefulReload request"
-        unless Fluent.windows?
+        if Fluent.windows?
+          supervisor_sigusr2_handler
+        else
           Process.kill :USR2, $$
         end
 
@@ -159,38 +170,101 @@ module Fluent
     end
 
     def install_supervisor_signal_handlers
+      return if Fluent.windows?
+
       trap :HUP do
         $log.debug "fluentd supervisor process get SIGHUP"
         supervisor_sighup_handler
-      end unless Fluent.windows?
+      end
 
       trap :USR1 do
         $log.debug "fluentd supervisor process get SIGUSR1"
         supervisor_sigusr1_handler
-      end unless Fluent.windows?
+      end
 
       trap :USR2 do
         $log.debug 'fluentd supervisor process got SIGUSR2'
         supervisor_sigusr2_handler
-      end unless Fluent.windows?
+      end
+    end
+
+    if Fluent.windows?
+      # Override some methods of ServerEngine::MultiSpawnWorker
+      # Since Fluentd's Supervisor doesn't use ServerEngine's HUP, USR1 and USR2
+      # handlers (see install_supervisor_signal_handlers), they should be
+      # disabled also on Windows, just send commands to workers instead.
+      def restart(graceful)
+        @monitors.each do |m|
+          m.send_command(graceful ? "GRACEFUL_RESTART\n" : "IMMEDIATE_RESTART\n")
+        end
+      end
+
+      def reload
+        @monitors.each do |m|
+          m.send_command("RELOAD\n")
+        end
+      end
     end
 
     def install_windows_event_handler
+      return unless Fluent.windows?
+
+      @pid_signame = "fluentd_#{$$}"
+      @signame = config[:signame]
+
       Thread.new do
-        ev = Win32::Event.new(@signame)
+        ipc = Win32::Ipc.new(nil)
+        events = [
+          Win32::Event.new("#{@pid_signame}_STOP_EVENT_THREAD"),
+          Win32::Event.new("#{@pid_signame}"),
+          Win32::Event.new("#{@pid_signame}_HUP"),
+          Win32::Event.new("#{@pid_signame}_USR1"),
+          Win32::Event.new("#{@pid_signame}_USR2"),
+        ]
+        if @signame
+          signame_events = [
+            Win32::Event.new("#{@signame}"),
+            Win32::Event.new("#{@signame}_HUP"),
+            Win32::Event.new("#{@signame}_USR1"),
+            Win32::Event.new("#{@signame}_USR2"),
+          ]
+          events.concat(signame_events)
+        end
         begin
-          ev.reset
-          until WaitForSingleObject(ev.handle, 0) == WAIT_OBJECT_0
-            sleep 1
+          loop do
+            idx = ipc.wait_any(events, Windows::Synchronize::INFINITE)
+            if idx > 0 && idx <= events.length
+              $log.debug("Got Win32 event \"#{events[idx - 1].name}\"")
+            else
+              $log.warn("Unexpected reutrn value of Win32::Ipc#wait_any: #{idx}")
+            end
+            case idx
+            when 2, 6
+              stop(true)
+            when 3, 7
+              supervisor_sighup_handler
+            when 4, 8
+              supervisor_sigusr1_handler
+            when 5, 9
+              supervisor_sigusr2_handler
+            when 1
+              break
+            end
           end
-          kill_worker
-          stop(true)
         ensure
-          ev.close
+          events.each { |event| event.close }
         end
       end
     end
 
+    def stop_windows_event_thread
+      if Fluent.windows?
+        ev = Win32::Event.open("#{@pid_signame}_STOP_EVENT_THREAD")
+        ev.set
+        ev.close
+      end
+    end
+
     def supervisor_sighup_handler
       kill_worker
     end
@@ -265,9 +339,25 @@ module Fluent
     def send_signal_to_workers(signal)
       return unless config[:worker_pid]
 
-      config[:worker_pid].each_value do |pid|
-        # don't rescue Errno::ESRCH here (invalid status)
-        Process.kill(signal, pid)
+      if Fluent.windows?
+        send_command_to_workers(signal)
+      else
+        config[:worker_pid].each_value do |pid|
+          # don't rescue Errno::ESRCH here (invalid status)
+          Process.kill(signal, pid)
+        end
+      end
+    end
+
+    def send_command_to_workers(signal)
+      # Use SeverEngine's CommandSender on Windows
+      case signal
+      when :HUP
+        restart(false)
+      when :USR1
+        restart(true)
+      when :USR2
+        reload
       end
     end
   end
@@ -302,6 +392,7 @@ module Fluent
       log_level = params['log_level']
       suppress_repeated_stacktrace = params['suppress_repeated_stacktrace']
       ignore_repeated_log_interval = params['ignore_repeated_log_interval']
+      ignore_same_log_interval = params['ignore_same_log_interval']
 
       log_path = params['log_path']
       chuser = params['chuser']
@@ -309,7 +400,8 @@ module Fluent
       log_rotate_age = params['log_rotate_age']
       log_rotate_size = params['log_rotate_size']
 
-      log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace, ignore_repeated_log_interval: ignore_repeated_log_interval}
+      log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace, ignore_repeated_log_interval: ignore_repeated_log_interval,
+                  ignore_same_log_interval: ignore_same_log_interval}
       logger_initializer = Supervisor::LoggerInitializer.new(
         log_path, log_level, chuser, chgroup, log_opts,
         log_rotate_age: log_rotate_age,
@@ -347,6 +439,7 @@ module Fluent
         chumask: 0,
         suppress_repeated_stacktrace: suppress_repeated_stacktrace,
         ignore_repeated_log_interval: ignore_repeated_log_interval,
+        ignore_same_log_interval: ignore_same_log_interval,
         daemonize: daemonize,
         rpc_endpoint: params['rpc_endpoint'],
         counter_server: params['counter_server'],
@@ -364,6 +457,7 @@ module Fluent
         config_path: path,
         main_cmd: params['main_cmd'],
         signame: params['signame'],
+        disable_shared_socket: params['disable_shared_socket']
       }
       if daemonize
         se_config[:pid_path] = pid_path
@@ -441,10 +535,11 @@ module Fluent
         self
       end
 
-      def apply_options(format: nil, time_format: nil, log_dir_perm: nil, ignore_repeated_log_interval: nil)
+      def apply_options(format: nil, time_format: nil, log_dir_perm: nil, ignore_repeated_log_interval: nil, ignore_same_log_interval: nil)
         $log.format = format if format
         $log.time_format = time_format if time_format
         $log.ignore_repeated_log_interval = ignore_repeated_log_interval if ignore_repeated_log_interval
+        $log.ignore_same_log_interval = ignore_same_log_interval if ignore_same_log_interval
 
         if @path && log_dir_perm
           File.chmod(log_dir_perm || 0755, File.dirname(@path))
@@ -478,7 +573,8 @@ module Fluent
         supervise: true,
         standalone_worker: false,
         signame: nil,
-        conf_encoding: 'utf-8'
+        conf_encoding: 'utf-8',
+        disable_shared_socket: nil
       }
     end
 
@@ -511,7 +607,8 @@ module Fluent
       @cl_opt = opt
       @conf = nil
 
-      log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval]}
+      log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval],
+                  ignore_same_log_interval: opt[:ignore_same_log_interval]}
       @log = LoggerInitializer.new(
         @log_path, opt[:log_level], @chuser, @chgroup, log_opts,
         log_rotate_age: @log_rotate_age,
@@ -635,7 +732,8 @@ module Fluent
         format: @system_config.log.format,
         time_format: @system_config.log.time_format,
         log_dir_perm: @system_config.dir_permission,
-        ignore_repeated_log_interval: @system_config.ignore_repeated_log_interval
+        ignore_repeated_log_interval: @system_config.ignore_repeated_log_interval,
+        ignore_same_log_interval: @system_config.ignore_same_log_interval
       )
 
       $log.info :supervisor, 'parsing config file is succeeded', path: @config_path
@@ -704,6 +802,7 @@ module Fluent
         'counter_server' => @system_config.counter_server,
         'log_format' => @system_config.log.format,
         'log_time_format' => @system_config.log.time_format,
+        'disable_shared_socket' => @system_config.disable_shared_socket
       }
 
       se = ServerEngine.create(ServerModule, WorkerModule){
@@ -740,33 +839,45 @@ module Fluent
         end
       end
 
-      trap :USR1 do
-        flush_buffer
-      end unless Fluent.windows?
+      if Fluent.windows?
+        install_main_process_command_handlers
+      else
+        trap :USR1 do
+          flush_buffer
+        end
 
-      trap :USR2 do
-        reload_config
-      end unless Fluent.windows?
+        trap :USR2 do
+          reload_config
+        end
+      end
+    end
 
-      if Fluent.windows?
-        command_pipe = STDIN.dup
-        STDIN.reopen(File::NULL, "rb")
-        command_pipe.binmode
-        command_pipe.sync = true
+    def install_main_process_command_handlers
+      command_pipe = $stdin.dup
+      $stdin.reopen(File::NULL, "rb")
+      command_pipe.binmode
+      command_pipe.sync = true
 
-        Thread.new do
-          loop do
-            cmd = command_pipe.gets.chomp
-            case cmd
-            when "GRACEFUL_STOP", "IMMEDIATE_STOP"
-              $log.debug "fluentd main process get #{cmd} command"
-              @finished = true
-              $log.debug "getting start to shutdown main process"
-              Fluent::Engine.stop
-              break
-            else
-              $log.warn "fluentd main process get unknown command [#{cmd}]"
-            end
+      Thread.new do
+        loop do
+          cmd = command_pipe.gets
+          break unless cmd
+
+          case cmd.chomp!
+          when "GRACEFUL_STOP", "IMMEDIATE_STOP"
+            $log.debug "fluentd main process get #{cmd} command"
+            @finished = true
+            $log.debug "getting start to shutdown main process"
+            Fluent::Engine.stop
+            break
+          when "GRACEFUL_RESTART"
+            $log.debug "fluentd main process get #{cmd} command"
+            flush_buffer
+          when "RELOAD"
+            $log.debug "fluentd main process get #{cmd} command"
+            reload_config
+          else
+            $log.warn "fluentd main process get unknown command [#{cmd}]"
           end
         end
       end