fluentd 1.11.3 → 1.12.0

data/lib/fluent/plugin/output.rb

@@ -769,17 +769,19 @@ module Fluent
             end
           end
 
-          if rvalue =~ CHUNK_KEY_PLACEHOLDER_PATTERN
-            log.warn "chunk key placeholder '#{$1}' not replaced. template:#{str}"
-          end
-
-          rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
+          rvalue = rvalue.sub(CHUNK_ID_PLACEHOLDER_PATTERN) {
             if chunk_passed
               dump_unique_id_hex(chunk.unique_id)
             else
               log.warn "${chunk_id} is not allowed in this plugin. Pass Chunk instead of metadata in extract_placeholders's 2nd argument"
             end
           }
+
+          if rvalue =~ CHUNK_KEY_PLACEHOLDER_PATTERN
+            log.warn "chunk key placeholder '#{$1}' not replaced. template:#{str}"
+          end
+
+          rvalue
         end
       end
 
@@ -1250,7 +1252,13 @@ module Fluent
             log.debug "buffer queue cleared"
             @retry = nil
           else
-            @retry.step
+            # Ensure that the current time is greater than or equal to @retry.next_time to avoid the situation when
+            # @retry.step is called almost as many times as the number of flush threads in a short time.
+            if Time.now >= @retry.next_time
+              @retry.step
+            else
+              @retry.recalc_next_time # to prevent all flush threads from retrying at the same time
+            end
             if error
               if using_secondary
                 msg = "failed to flush the buffer with secondary output."

data/lib/fluent/plugin_helper/http_server/compat/server.rb

@@ -81,7 +81,7 @@ module Fluent
 
           def build_handler
             @methods.group_by(&:first).each do |(path, rest)|
-              klass = Fluent::PluginHelper::HttpServer::Compat::WebrickHandler.build(Hash[rest.map { |e| [e[1], e[2]] }])
+              klass = Fluent::PluginHelper::HttpServer::Compat::WebrickHandler.build(**Hash[rest.map { |e| [e[1], e[2]] }])
               @server.mount(path, klass)
             end
           end

data/lib/fluent/plugin_helper/inject.rb

@@ -76,7 +76,7 @@ module Fluent
           config_param :time_key, :string, default: nil
 
           # To avoid defining :time_type twice
-          config_param :time_type, :enum, list: [:float, :unixtime, :string], default: :float
+          config_param :time_type, :enum, list: [:float, :unixtime, :unixtime_millis, :unixtime_micros, :unixtime_nanos, :string], default: :float
 
           Fluent::TimeMixin::TIME_PARAMETERS.each do |name, type, opts|
             config_param(name, type, **opts)
@@ -132,6 +132,9 @@ module Fluent
           if @_inject_time_key
             @_inject_time_formatter = case @inject_config.time_type
                                       when :float then ->(time){ time.to_r.truncate(+6).to_f } # microsecond floating point value
+                                      when :unixtime_millis then ->(time) { time.respond_to?(:nsec) ? time.to_i * 1_000 + time.nsec / 1_000_000 : (time * 1_000).floor }
+                                      when :unixtime_micros then ->(time) { time.respond_to?(:nsec) ? time.to_i * 1_000_000 + time.nsec / 1_000 : (time * 1_000_000).floor }
+                                      when :unixtime_nanos then ->(time) { time.respond_to?(:nsec) ? time.to_i * 1_000_000_000 + time.nsec : (time * 1_000_000_000).floor }
                                       when :unixtime then ->(time){ time.to_i }
                                       else
                                         localtime = @inject_config.localtime && !@inject_config.utc

data/lib/fluent/plugin_helper/retry_state.rb

@@ -127,6 +127,10 @@ module Fluent
           nil
         end
 
+        def recalc_next_time
+          @next_time = calc_next_time
+        end
+
         def limit?
           if @forever
             false

data/lib/fluent/supervisor.rb

@@ -53,11 +53,11 @@ module Fluent
         @enable_get_dump = config[:enable_get_dump]
         run_rpc_server
       end
-      install_supervisor_signal_handlers
 
-      if config[:signame]
-        @signame = config[:signame]
+      if Fluent.windows?
         install_windows_event_handler
+      else
+        install_supervisor_signal_handlers
       end
 
       if counter = config[:counter_server]
@@ -70,6 +70,7 @@ module Fluent
     end
 
     def after_run
+      stop_windows_event_thread if Fluent.windows?
       stop_rpc_server if @rpc_endpoint
       stop_counter_server if @counter
       Fluent::Supervisor.cleanup_resources
@@ -92,7 +93,8 @@ module Fluent
       @rpc_server.mount_proc('/api/processes.flushBuffersAndKillWorkers') { |req, res|
         $log.debug "fluentd RPC got /api/processes.flushBuffersAndKillWorkers request"
         if Fluent.windows?
-          $log.warn "operation 'flushBuffersAndKillWorkers' is not supported on Windows now."
+          supervisor_sigusr1_handler
+          stop(true)
         else
           Process.kill :USR1, $$
           Process.kill :TERM, $$
@@ -101,7 +103,9 @@ module Fluent
       }
       @rpc_server.mount_proc('/api/plugins.flushBuffers') { |req, res|
         $log.debug "fluentd RPC got /api/plugins.flushBuffers request"
-        unless Fluent.windows?
+        if Fluent.windows?
+          supervisor_sigusr1_handler
+        else
           Process.kill :USR1, $$
         end
         nil
@@ -125,7 +129,9 @@ module Fluent
 
       @rpc_server.mount_proc('/api/config.gracefulReload') { |req, res|
         $log.debug "fluentd RPC got /api/config.gracefulReload request"
-        unless Fluent.windows?
+        if Fluent.windows?
+          supervisor_sigusr2_handler
+        else
           Process.kill :USR2, $$
         end
 
@@ -159,38 +165,101 @@ module Fluent
     end
 
     def install_supervisor_signal_handlers
+      return if Fluent.windows?
+
       trap :HUP do
         $log.debug "fluentd supervisor process get SIGHUP"
         supervisor_sighup_handler
-      end unless Fluent.windows?
+      end
 
       trap :USR1 do
         $log.debug "fluentd supervisor process get SIGUSR1"
         supervisor_sigusr1_handler
-      end unless Fluent.windows?
+      end
 
       trap :USR2 do
         $log.debug 'fluentd supervisor process got SIGUSR2'
         supervisor_sigusr2_handler
-      end unless Fluent.windows?
+      end
+    end
+
+    if Fluent.windows?
+      # Override some methods of ServerEngine::MultiSpawnWorker
+      # Since Fluentd's Supervisor doesn't use ServerEngine's HUP, USR1 and USR2
+      # handlers (see install_supervisor_signal_handlers), they should be
+      # disabled also on Windows, just send commands to workers instead.
+      def restart(graceful)
+        @monitors.each do |m|
+          m.send_command(graceful ? "GRACEFUL_RESTART\n" : "IMMEDIATE_RESTART\n")
+        end
+      end
+
+      def reload
+        @monitors.each do |m|
+          m.send_command("RELOAD\n")
+        end
+      end
     end
 
     def install_windows_event_handler
+      return unless Fluent.windows?
+
+      @pid_signame = "fluentd_#{$$}"
+      @signame = config[:signame]
+
       Thread.new do
-        ev = Win32::Event.new(@signame)
+        ipc = Win32::Ipc.new(nil)
+        events = [
+          Win32::Event.new("#{@pid_signame}_STOP_EVENT_THREAD"),
+          Win32::Event.new("#{@pid_signame}"),
+          Win32::Event.new("#{@pid_signame}_HUP"),
+          Win32::Event.new("#{@pid_signame}_USR1"),
+          Win32::Event.new("#{@pid_signame}_USR2"),
+        ]
+        if @signame
+          signame_events = [
+            Win32::Event.new("#{@signame}"),
+            Win32::Event.new("#{@signame}_HUP"),
+            Win32::Event.new("#{@signame}_USR1"),
+            Win32::Event.new("#{@signame}_USR2"),
+          ]
+          events.concat(signame_events)
+        end
         begin
-          ev.reset
-          until WaitForSingleObject(ev.handle, 0) == WAIT_OBJECT_0
-            sleep 1
+          loop do
+            idx = ipc.wait_any(events, Windows::Synchronize::INFINITE)
+            if idx > 0 && idx <= events.length
+              $log.debug("Got Win32 event \"#{events[idx - 1].name}\"")
+            else
+              $log.warn("Unexpected reutrn value of Win32::Ipc#wait_any: #{idx}")
+            end
+            case idx
+            when 2, 6
+              stop(true)
+            when 3, 7
+              supervisor_sighup_handler
+            when 4, 8
+              supervisor_sigusr1_handler
+            when 5, 9
+              supervisor_sigusr2_handler
+            when 1
+              break
+            end
           end
-          kill_worker
-          stop(true)
         ensure
-          ev.close
+          events.each { |event| event.close }
         end
       end
     end
 
+    def stop_windows_event_thread
+      if Fluent.windows?
+        ev = Win32::Event.open("#{@pid_signame}_STOP_EVENT_THREAD")
+        ev.set
+        ev.close
+      end
+    end
+
     def supervisor_sighup_handler
       kill_worker
     end
@@ -265,9 +334,25 @@ module Fluent
     def send_signal_to_workers(signal)
       return unless config[:worker_pid]
 
-      config[:worker_pid].each_value do |pid|
-        # don't rescue Errno::ESRCH here (invalid status)
-        Process.kill(signal, pid)
+      if Fluent.windows?
+        send_command_to_workers(signal)
+      else
+        config[:worker_pid].each_value do |pid|
+          # don't rescue Errno::ESRCH here (invalid status)
+          Process.kill(signal, pid)
+        end
+      end
+    end
+
+    def send_command_to_workers(signal)
+      # Use SeverEngine's CommandSender on Windows
+      case signal
+      when :HUP
+        restart(false)
+      when :USR1
+        restart(true)
+      when :USR2
+        reload
       end
     end
   end
@@ -746,33 +831,45 @@ module Fluent
         end
       end
 
-      trap :USR1 do
-        flush_buffer
-      end unless Fluent.windows?
+      if Fluent.windows?
+        install_main_process_command_handlers
+      else
+        trap :USR1 do
+          flush_buffer
+        end
 
-      trap :USR2 do
-        reload_config
-      end unless Fluent.windows?
+        trap :USR2 do
+          reload_config
+        end
+      end
+    end
 
-      if Fluent.windows?
-        command_pipe = STDIN.dup
-        STDIN.reopen(File::NULL, "rb")
-        command_pipe.binmode
-        command_pipe.sync = true
+    def install_main_process_command_handlers
+      command_pipe = $stdin.dup
+      $stdin.reopen(File::NULL, "rb")
+      command_pipe.binmode
+      command_pipe.sync = true
 
-        Thread.new do
-          loop do
-            cmd = command_pipe.gets.chomp
-            case cmd
-            when "GRACEFUL_STOP", "IMMEDIATE_STOP"
-              $log.debug "fluentd main process get #{cmd} command"
-              @finished = true
-              $log.debug "getting start to shutdown main process"
-              Fluent::Engine.stop
-              break
-            else
-              $log.warn "fluentd main process get unknown command [#{cmd}]"
-            end
+      Thread.new do
+        loop do
+          cmd = command_pipe.gets
+          break unless cmd
+
+          case cmd.chomp!
+          when "GRACEFUL_STOP", "IMMEDIATE_STOP"
+            $log.debug "fluentd main process get #{cmd} command"
+            @finished = true
+            $log.debug "getting start to shutdown main process"
+            Fluent::Engine.stop
+            break
+          when "GRACEFUL_RESTART"
+            $log.debug "fluentd main process get #{cmd} command"
+            flush_buffer
+          when "RELOAD"
+            $log.debug "fluentd main process get #{cmd} command"
+            reload_config
+          else
+            $log.warn "fluentd main process get unknown command [#{cmd}]"
           end
         end
       end

data/lib/fluent/time.rb

@@ -50,6 +50,7 @@ module Fluent
     def to_int
       @sec
     end
+    alias :to_i :to_int
 
     def to_f
       @sec + @nsec / 1_000_000_000.0

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.11.3'
+  VERSION = '1.12.0'
 
 end

data/lib/fluent/winsvc.rb

@@ -61,7 +61,6 @@ begin
     end
     
     def service_main
-    
       @pid = service_main_start(@service_name)
       while running?
         sleep 10
@@ -69,13 +68,32 @@ begin
     end
 
     def service_stop
-      ev = Win32::Event.open(@service_name)
-      ev.set
-      ev.close
+      set_event(@service_name)
       if @pid > 0
         Process.waitpid(@pid)
       end
     end
+
+    def service_paramchange
+      set_event("#{@service_name}_USR2")
+    end
+
+    def service_user_defined_control(code)
+      case code
+      when 128
+        set_event("#{@service_name}_HUP")
+      when 129
+        set_event("#{@service_name}_USR1")
+      end
+    end
+
+    private
+
+    def set_event(event_name)
+      ev = Win32::Event.open(event_name)
+      ev.set
+      ev.close
+    end
   end
 
   FluentdService.new(opts[:service_name]).mainloop

data/test/command/test_binlog_reader.rb

@@ -82,6 +82,14 @@ class TestBaseCommand < ::Test::Unit::TestCase
 end
 
 class TestHead < TestBaseCommand
+  setup do
+    @default_newline = if Fluent.windows?
+                         "\r\n"
+                       else
+                         "\n"
+                       end
+  end
+
   sub_test_case 'initialize' do
     data(
       'file is not passed' => %w(),
@@ -138,7 +146,7 @@ class TestHead < TestBaseCommand
         create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
         head = BinlogReaderCommand::Head.new(argv)
         out = capture_stdout { head.call }
-        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}\n" * 5, out
+        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}#{@default_newline}" * 5, out
       end
     end
 
@@ -149,7 +157,7 @@ class TestHead < TestBaseCommand
         create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
         head = BinlogReaderCommand::Head.new(argv)
         out = capture_stdout { head.call }
-        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}\n", out
+        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}#{@default_newline}", out
       end
     end
 
@@ -169,7 +177,7 @@ class TestHead < TestBaseCommand
         create_message_packed_file(@file_name, [event_time(@t).to_i], [@record])
         head = BinlogReaderCommand::Head.new(argv)
         out = capture_stdout { head.call }
-        assert_equal "#{Yajl.dump(@record)}\n", out
+        assert_equal "#{Yajl.dump(@record)}#{@default_newline}", out
       end
     end
 
@@ -198,6 +206,14 @@ class TestHead < TestBaseCommand
 end
 
 class TestCat < TestBaseCommand
+  setup do
+    @default_newline = if Fluent.windows?
+                         "\r\n"
+                       else
+                         "\n"
+                       end
+  end
+
   sub_test_case 'initialize' do
     data(
       'file is not passed' => [],
@@ -254,7 +270,7 @@ class TestCat < TestBaseCommand
         create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
         head = BinlogReaderCommand::Cat.new(argv)
         out = capture_stdout { head.call }
-        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}\n" * 6, out
+        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}#{@default_newline}" * 6, out
       end
     end
 
@@ -265,7 +281,7 @@ class TestCat < TestBaseCommand
         create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
         head = BinlogReaderCommand::Cat.new(argv)
         out = capture_stdout { head.call }
-        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}\n", out
+        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}#{@default_newline}", out
       end
     end
 
@@ -276,7 +292,7 @@ class TestCat < TestBaseCommand
         create_message_packed_file(@file_name, [event_time(@t).to_i], [@record])
         head = BinlogReaderCommand::Cat.new(argv)
         out = capture_stdout { head.call }
-        assert_equal "#{Yajl.dump(@record)}\n", out
+        assert_equal "#{Yajl.dump(@record)}#{@default_newline}", out
       end
     end