fluentd 1.12.0.rc1 → 1.12.0.rc2

data/lib/fluent/plugin/in_tail/position_file.rb

@@ -22,34 +22,40 @@ module Fluent::Plugin
       UNWATCHED_POSITION = 0xffffffffffffffff
       POSITION_FILE_ENTRY_REGEX = /^([^\t]+)\t([0-9a-fA-F]+)\t([0-9a-fA-F]+)/.freeze
 
-      def self.load(file, logger:)
-        pf = new(file, logger: logger)
+      def self.load(file, follow_inodes, existing_paths, logger:)
+        pf = new(file, follow_inodes, existing_paths, logger: logger)
         pf.load
         pf
       end
 
-      def initialize(file, logger: nil)
+      def initialize(file, follow_inodes, existing_paths, logger: nil)
         @file = file
         @logger = logger
         @file_mutex = Mutex.new
         @map = {}
+        @follow_inodes = follow_inodes
+        @existing_paths = existing_paths
       end
 
-      def [](path)
-        if m = @map[path]
+      def [](target_info)
+        if m = @map[@follow_inodes ? target_info.ino : target_info.path]
           return m
         end
 
         @file_mutex.synchronize {
           @file.seek(0, IO::SEEK_END)
-          seek = @file.pos + path.bytesize + 1
-          @file.write "#{path}\t0000000000000000\t0000000000000000\n"
-          @map[path] = FilePositionEntry.new(@file, @file_mutex, seek, 0, 0)
+          seek = @file.pos + target_info.path.bytesize + 1
+          @file.write "#{target_info.path}\t0000000000000000\t0000000000000000\n"
+          if @follow_inodes
+            @map[target_info.ino] = FilePositionEntry.new(@file, @file_mutex, seek, 0, 0)
+          else
+            @map[target_info.path] = FilePositionEntry.new(@file, @file_mutex, seek, 0, 0)
+          end
         }
       end
 
-      def unwatch(path)
-        if (entry = @map.delete(path))
+      def unwatch(target_info)
+        if (entry = @map.delete(@follow_inodes ? target_info.ino : target_info.path))
           entry.update_pos(UNWATCHED_POSITION)
         end
       end
@@ -69,7 +75,11 @@ module Fluent::Plugin
             pos = m[2].to_i(16)
             ino = m[3].to_i(16)
             seek = @file.pos - line.bytesize + path.bytesize + 1
-            map[path] = FilePositionEntry.new(@file, @file_mutex, seek, pos, ino)
+            if @follow_inodes
+              map[ino] = FilePositionEntry.new(@file, @file_mutex, seek, pos, ino)
+            else
+              map[path] = FilePositionEntry.new(@file, @file_mutex, seek, pos, ino)
+            end
           end
         end
 
@@ -94,8 +104,9 @@ module Fluent::Plugin
             @file.truncate(0)
             @file.write(entries.values.map(&:to_entry_fmt).join)
 
-            entries.each do |path, val|
-              if (m = @map[path])
+            # entry contains path/ino key and value.
+            entries.each do |key, val|
+              if (m = @map[key])
                 m.seek = val.seek
               end
             end
@@ -139,13 +150,25 @@ module Fluent::Plugin
               @logger.warn("#{path} already exists. use latest one: deleted #{entries[path]}") if @logger
             end
 
-            entries[path] = Entry.new(path, pos, ino, file_pos + path.size + 1)
+            if @follow_inodes
+              entries[ino] = Entry.new(path, pos, ino, file_pos + path.size + 1)
+            else
+              entries[path] = Entry.new(path, pos, ino, file_pos + path.size + 1)
+            end
             file_pos += line.size
           end
         end
 
+        entries = remove_deleted_files_entries(entries, @existing_paths) if @follow_inodes
         entries
       end
+
+      def remove_deleted_files_entries(existent_entries, existing_paths)
+        filtered_entries = existent_entries.select {|file_entry|
+          existing_paths.key?(file_entry)
+        }
+        filtered_entries
+      end
     end
 
     Entry = Struct.new(:path, :pos, :ino, :seek) do
@@ -224,5 +247,7 @@ module Fluent::Plugin
         @inode
       end
     end
+
+    TargetInfo = Struct.new(:path, :ino)
   end
 end

data/lib/fluent/plugin/output.rb

@@ -1252,7 +1252,13 @@ module Fluent
             log.debug "buffer queue cleared"
             @retry = nil
           else
-            @retry.step
+            # Ensure that the current time is greater than or equal to @retry.next_time to avoid the situation when
+            # @retry.step is called almost as many times as the number of flush threads in a short time.
+            if Time.now >= @retry.next_time
+              @retry.step
+            else
+              @retry.recalc_next_time # to prevent all flush threads from retrying at the same time
+            end
             if error
               if using_secondary
                 msg = "failed to flush the buffer with secondary output."

data/lib/fluent/plugin_helper/http_server/compat/server.rb

@@ -81,7 +81,7 @@ module Fluent
 
           def build_handler
             @methods.group_by(&:first).each do |(path, rest)|
-              klass = Fluent::PluginHelper::HttpServer::Compat::WebrickHandler.build(Hash[rest.map { |e| [e[1], e[2]] }])
+              klass = Fluent::PluginHelper::HttpServer::Compat::WebrickHandler.build(**Hash[rest.map { |e| [e[1], e[2]] }])
               @server.mount(path, klass)
             end
           end

data/lib/fluent/plugin_helper/retry_state.rb

@@ -127,6 +127,10 @@ module Fluent
           nil
         end
 
+        def recalc_next_time
+          @next_time = calc_next_time
+        end
+
         def limit?
           if @forever
             false

data/lib/fluent/supervisor.rb

@@ -53,11 +53,11 @@ module Fluent
         @enable_get_dump = config[:enable_get_dump]
         run_rpc_server
       end
-      install_supervisor_signal_handlers
 
-      if config[:signame]
-        @signame = config[:signame]
+      if Fluent.windows?
         install_windows_event_handler
+      else
+        install_supervisor_signal_handlers
       end
 
       if counter = config[:counter_server]
@@ -70,6 +70,7 @@ module Fluent
     end
 
     def after_run
+      stop_windows_event_thread if Fluent.windows?
       stop_rpc_server if @rpc_endpoint
       stop_counter_server if @counter
       Fluent::Supervisor.cleanup_resources
@@ -92,7 +93,8 @@ module Fluent
       @rpc_server.mount_proc('/api/processes.flushBuffersAndKillWorkers') { |req, res|
         $log.debug "fluentd RPC got /api/processes.flushBuffersAndKillWorkers request"
         if Fluent.windows?
-          $log.warn "operation 'flushBuffersAndKillWorkers' is not supported on Windows now."
+          supervisor_sigusr1_handler
+          stop(true)
         else
           Process.kill :USR1, $$
           Process.kill :TERM, $$
@@ -101,7 +103,9 @@ module Fluent
       }
       @rpc_server.mount_proc('/api/plugins.flushBuffers') { |req, res|
         $log.debug "fluentd RPC got /api/plugins.flushBuffers request"
-        unless Fluent.windows?
+        if Fluent.windows?
+          supervisor_sigusr1_handler
+        else
           Process.kill :USR1, $$
         end
         nil
@@ -125,7 +129,9 @@ module Fluent
 
       @rpc_server.mount_proc('/api/config.gracefulReload') { |req, res|
         $log.debug "fluentd RPC got /api/config.gracefulReload request"
-        unless Fluent.windows?
+        if Fluent.windows?
+          supervisor_sigusr2_handler
+        else
           Process.kill :USR2, $$
         end
 
@@ -159,37 +165,101 @@ module Fluent
     end
 
     def install_supervisor_signal_handlers
+      return if Fluent.windows?
+
       trap :HUP do
         $log.debug "fluentd supervisor process get SIGHUP"
         supervisor_sighup_handler
-      end unless Fluent.windows?
+      end
 
       trap :USR1 do
         $log.debug "fluentd supervisor process get SIGUSR1"
         supervisor_sigusr1_handler
-      end unless Fluent.windows?
+      end
 
       trap :USR2 do
         $log.debug 'fluentd supervisor process got SIGUSR2'
         supervisor_sigusr2_handler
-      end unless Fluent.windows?
+      end
+    end
+
+    if Fluent.windows?
+      # Override some methods of ServerEngine::MultiSpawnWorker
+      # Since Fluentd's Supervisor doesn't use ServerEngine's HUP, USR1 and USR2
+      # handlers (see install_supervisor_signal_handlers), they should be
+      # disabled also on Windows, just send commands to workers instead.
+      def restart(graceful)
+        @monitors.each do |m|
+          m.send_command(graceful ? "GRACEFUL_RESTART\n" : "IMMEDIATE_RESTART\n")
+        end
+      end
+
+      def reload
+        @monitors.each do |m|
+          m.send_command("RELOAD\n")
+        end
+      end
     end
 
     def install_windows_event_handler
+      return unless Fluent.windows?
+
+      @pid_signame = "fluentd_#{$$}"
+      @signame = config[:signame]
+
       Thread.new do
-        ev = Win32::Event.new(@signame)
+        ipc = Win32::Ipc.new(nil)
+        events = [
+          Win32::Event.new("#{@pid_signame}_STOP_EVENT_THREAD"),
+          Win32::Event.new("#{@pid_signame}"),
+          Win32::Event.new("#{@pid_signame}_HUP"),
+          Win32::Event.new("#{@pid_signame}_USR1"),
+          Win32::Event.new("#{@pid_signame}_USR2"),
+        ]
+        if @signame
+          signame_events = [
+            Win32::Event.new("#{@signame}"),
+            Win32::Event.new("#{@signame}_HUP"),
+            Win32::Event.new("#{@signame}_USR1"),
+            Win32::Event.new("#{@signame}_USR2"),
+          ]
+          events.concat(signame_events)
+        end
         begin
-          ev.reset
-          until WaitForSingleObject(ev.handle, 0) == WAIT_OBJECT_0
-            sleep 1
+          loop do
+            idx = ipc.wait_any(events, Windows::Synchronize::INFINITE)
+            if idx > 0 && idx <= events.length
+              $log.debug("Got Win32 event \"#{events[idx - 1].name}\"")
+            else
+              $log.warn("Unexpected reutrn value of Win32::Ipc#wait_any: #{idx}")
+            end
+            case idx
+            when 2, 6
+              stop(true)
+            when 3, 7
+              supervisor_sighup_handler
+            when 4, 8
+              supervisor_sigusr1_handler
+            when 5, 9
+              supervisor_sigusr2_handler
+            when 1
+              break
+            end
           end
-          stop(true)
         ensure
-          ev.close
+          events.each { |event| event.close }
         end
       end
     end
 
+    def stop_windows_event_thread
+      if Fluent.windows?
+        ev = Win32::Event.open("#{@pid_signame}_STOP_EVENT_THREAD")
+        ev.set
+        ev.close
+      end
+    end
+
     def supervisor_sighup_handler
       kill_worker
     end
@@ -264,9 +334,25 @@ module Fluent
     def send_signal_to_workers(signal)
       return unless config[:worker_pid]
 
-      config[:worker_pid].each_value do |pid|
-        # don't rescue Errno::ESRCH here (invalid status)
-        Process.kill(signal, pid)
+      if Fluent.windows?
+        send_command_to_workers(signal)
+      else
+        config[:worker_pid].each_value do |pid|
+          # don't rescue Errno::ESRCH here (invalid status)
+          Process.kill(signal, pid)
+        end
+      end
+    end
+
+    def send_command_to_workers(signal)
+      # Use SeverEngine's CommandSender on Windows
+      case signal
+      when :HUP
+        restart(false)
+      when :USR1
+        restart(true)
+      when :USR2
+        reload
       end
     end
   end
@@ -745,33 +831,45 @@ module Fluent
         end
       end
 
-      trap :USR1 do
-        flush_buffer
-      end unless Fluent.windows?
+      if Fluent.windows?
+        install_main_process_command_handlers
+      else
+        trap :USR1 do
+          flush_buffer
+        end
 
-      trap :USR2 do
-        reload_config
-      end unless Fluent.windows?
+        trap :USR2 do
+          reload_config
+        end
+      end
+    end
 
-      if Fluent.windows?
-        command_pipe = STDIN.dup
-        STDIN.reopen(File::NULL, "rb")
-        command_pipe.binmode
-        command_pipe.sync = true
+    def install_main_process_command_handlers
+      command_pipe = $stdin.dup
+      $stdin.reopen(File::NULL, "rb")
+      command_pipe.binmode
+      command_pipe.sync = true
 
-        Thread.new do
-          loop do
-            cmd = command_pipe.gets.chomp
-            case cmd
-            when "GRACEFUL_STOP", "IMMEDIATE_STOP"
-              $log.debug "fluentd main process get #{cmd} command"
-              @finished = true
-              $log.debug "getting start to shutdown main process"
-              Fluent::Engine.stop
-              break
-            else
-              $log.warn "fluentd main process get unknown command [#{cmd}]"
-            end
+      Thread.new do
+        loop do
+          cmd = command_pipe.gets
+          break unless cmd
+
+          case cmd.chomp!
+          when "GRACEFUL_STOP", "IMMEDIATE_STOP"
+            $log.debug "fluentd main process get #{cmd} command"
+            @finished = true
+            $log.debug "getting start to shutdown main process"
+            Fluent::Engine.stop
+            break
+          when "GRACEFUL_RESTART"
+            $log.debug "fluentd main process get #{cmd} command"
+            flush_buffer
+          when "RELOAD"
+            $log.debug "fluentd main process get #{cmd} command"
+            reload_config
+          else
+            $log.warn "fluentd main process get unknown command [#{cmd}]"
           end
         end
       end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.12.0.rc1'
+  VERSION = '1.12.0.rc2'
 
 end

data/lib/fluent/winsvc.rb

@@ -61,7 +61,6 @@ begin
     end
     
     def service_main
-    
       @pid = service_main_start(@service_name)
       while running?
         sleep 10
@@ -69,13 +68,32 @@ begin
     end
 
     def service_stop
-      ev = Win32::Event.open(@service_name)
-      ev.set
-      ev.close
+      set_event(@service_name)
       if @pid > 0
         Process.waitpid(@pid)
       end
     end
+
+    def service_paramchange
+      set_event("#{@service_name}_USR2")
+    end
+
+    def service_user_defined_control(code)
+      case code
+      when 128
+        set_event("#{@service_name}_HUP")
+      when 129
+        set_event("#{@service_name}_USR1")
+      end
+    end
+
+    private
+
+    def set_event(event_name)
+      ev = Win32::Event.open(event_name)
+      ev.set
+      ev.close
+    end
   end
 
   FluentdService.new(opts[:service_name]).mainloop