fluentd 1.14.5-x86-mingw32 → 1.15.1-x86-mingw32

data/lib/fluent/plugin/parser_syslog.rb

@@ -484,7 +484,7 @@ module Fluent
 
         time = begin
                  @time_parser_rfc5424.parse(time_str)
-               rescue Fluent::TimeParser::TimeParseError => e
+               rescue Fluent::TimeParser::TimeParseError
                  @time_parser_rfc5424_without_subseconds.parse(time_str)
                end
         record['time'] = time_str if @keep_time_key

data/lib/fluent/plugin_helper/retry_state.rb

@@ -44,6 +44,8 @@ module Fluent
 
           @timeout = timeout
           @timeout_at = @start + timeout
+          @has_reached_timeout = false
+          @has_timed_out = false
           @current = :primary
 
           if randomize_width < 0 || randomize_width > 0.5
@@ -98,7 +100,7 @@ module Fluent
               naive
             end
           elsif @current == :secondary
-            naive = naive_next_time(@steps - @secondary_transition_steps + 1)
+            naive = naive_next_time(@steps - @secondary_transition_steps)
             if naive >= @timeout_at
               @timeout_at
             else
@@ -123,7 +125,15 @@ module Fluent
             @current = :secondary
             @secondary_transition_steps = @steps
           end
+
           @next_time = calc_next_time
+
+          if @has_reached_timeout
+            @has_timed_out = @next_time >= @timeout_at
+          else
+            @has_reached_timeout = @next_time >= @timeout_at
+          end
+
           nil
         end
 
@@ -135,7 +145,7 @@ module Fluent
           if @forever
             false
           else
-            @next_time >= @timeout_at || !!(@max_steps && @steps >= @max_steps)
+            @has_timed_out || !!(@max_steps && @steps >= @max_steps)
           end
         end
       end
@@ -165,7 +175,7 @@ module Fluent
         end
 
         def calc_interval(num)
-          interval = raw_interval(num - 1)
+          interval = raw_interval(num)
           if @max_interval && interval > @max_interval
             @max_interval
           else
@@ -175,7 +185,7 @@ module Fluent
               # Calculate previous finite value to avoid inf related errors. If this re-computing is heavy, use cache.
               until interval.finite?
                 num -= 1
-                interval = raw_interval(num - 1)
+                interval = raw_interval(num)
               end
               interval
             end

data/lib/fluent/plugin_helper/server.rb

@@ -80,8 +80,8 @@ module Fluent
         raise ArgumentError, "BUG: block not specified which handles connection" unless block_given?
         raise ArgumentError, "BUG: block must have just one argument" unless block.arity == 1
 
-        if proto == :tcp || proto == :tls # default linger_timeout only for server
-          socket_options[:linger_timeout] ||= 0
+        if proto == :tcp || proto == :tls
+          socket_options[:linger_timeout] ||= @transport_config&.linger_timeout || 0
         end
 
         socket_option_validate!(proto, **socket_options)
@@ -132,8 +132,8 @@ module Fluent
         raise ArgumentError, "BUG: block not specified which handles received data" unless block_given?
         raise ArgumentError, "BUG: block must have 1 or 2 arguments" unless callback.arity == 1 || callback.arity == 2
 
-        if proto == :tcp || proto == :tls # default linger_timeout only for server
-          socket_options[:linger_timeout] ||= 0
+        if proto == :tcp || proto == :tls
+          socket_options[:linger_timeout] ||= @transport_config&.linger_timeout || 0
         end
 
         unless socket
@@ -263,6 +263,25 @@ module Fluent
         include Fluent::Configurable
         config_section :transport, required: false, multi: false, init: true, param_name: :transport_config do
           config_argument :protocol, :enum, list: [:tcp, :tls], default: :tcp
+
+          ### Socket Params ###
+
+          # SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src.
+          # Set positive value if needing to send FIN on closing on non-Windows.
+          # (On Windows, Fluentd can send FIN with zero `linger_timeout` since Fluentd doesn't set 0 to SO_LINGER on Windows.
+          # See `socket_option.rb`.)
+          # NOTE:
+            # Socket-options can be specified from each plugin as needed, so most of them is not defined here for now.
+            # This is because there is no positive reason to do so.
+            # `linger_timeout` option in particular needs to be defined here
+            # although it can be specified from each plugin as well.
+            # This is because this helper fixes the default value to `0` for its own reason
+            # and it has a critical effect on the behavior.
+          desc 'The timeout time used to set linger option.'
+          config_param :linger_timeout, :integer, default: 0
+
+          ### TLS Params ###
+
           config_param :version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: Fluent::TLS::DEFAULT_VERSION
           config_param :min_version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: nil
           config_param :max_version, :enum, list: Fluent::TLS::SUPPORTED_VERSIONS, default: nil

data/lib/fluent/plugin_helper/service_discovery.rb

@@ -44,7 +44,7 @@ module Fluent
 
         @discovery_manager.start
         unless @discovery_manager.static_config?
-          timer_execute(@_plugin_helper_service_discovery_title, @_plugin_helper_service_discovery_iterval) do
+          timer_execute(@_plugin_helper_service_discovery_title, @_plugin_helper_service_discovery_interval) do
             @discovery_manager.run_once
           end
         end
@@ -96,7 +96,7 @@ module Fluent
       # @param custom_build_method [Proc]
       def service_discovery_create_manager(title, configurations:, load_balancer: nil, custom_build_method: nil, interval: 3)
         @_plugin_helper_service_discovery_title = title
-        @_plugin_helper_service_discovery_iterval = interval
+        @_plugin_helper_service_discovery_interval = interval
 
         @discovery_manager = Fluent::PluginHelper::ServiceDiscovery::Manager.new(
           log: log,

data/lib/fluent/rpc.rb

@@ -20,9 +20,10 @@ module Fluent
   module RPC
     class Server
       def initialize(endpoint, log)
-        bind, port = endpoint.split(':')
-        @bind = bind
-        @port = port
+        m = endpoint.match(/^\[?(?<host>[0-9a-zA-Z:\-\.]+)\]?:(?<port>[0-9]+)$/)
+        raise Fluent::ConfigError, "Invalid rpc_endpoint: #{endpoint}" unless m
+        @bind = m[:host]
+        @port = m[:port]
         @log = log
 
         @server = WEBrick::HTTPServer.new(

data/lib/fluent/supervisor.rb

@@ -16,6 +16,7 @@
 
 require 'fileutils'
 require 'open3'
+require 'pathname'
 
 require 'fluent/config'
 require 'fluent/counter'
@@ -215,44 +216,50 @@ module Fluent
       Thread.new do
         ipc = Win32::Ipc.new(nil)
         events = [
-          Win32::Event.new("#{@pid_signame}_STOP_EVENT_THREAD"),
-          Win32::Event.new("#{@pid_signame}"),
-          Win32::Event.new("#{@pid_signame}_HUP"),
-          Win32::Event.new("#{@pid_signame}_USR1"),
-          Win32::Event.new("#{@pid_signame}_USR2"),
+          {win32_event: Win32::Event.new("#{@pid_signame}_STOP_EVENT_THREAD"), action: :stop_event_thread},
+          {win32_event: Win32::Event.new("#{@pid_signame}"), action: :stop},
+          {win32_event: Win32::Event.new("#{@pid_signame}_HUP"), action: :hup},
+          {win32_event: Win32::Event.new("#{@pid_signame}_USR1"), action: :usr1},
+          {win32_event: Win32::Event.new("#{@pid_signame}_USR2"), action: :usr2},
+          {win32_event: Win32::Event.new("#{@pid_signame}_CONT"), action: :cont},
         ]
         if @signame
           signame_events = [
-            Win32::Event.new("#{@signame}"),
-            Win32::Event.new("#{@signame}_HUP"),
-            Win32::Event.new("#{@signame}_USR1"),
-            Win32::Event.new("#{@signame}_USR2"),
+            {win32_event: Win32::Event.new("#{@signame}"), action: :stop},
+            {win32_event: Win32::Event.new("#{@signame}_HUP"), action: :hup},
+            {win32_event: Win32::Event.new("#{@signame}_USR1"), action: :usr1},
+            {win32_event: Win32::Event.new("#{@signame}_USR2"), action: :usr2},
+            {win32_event: Win32::Event.new("#{@signame}_CONT"), action: :cont},
           ]
           events.concat(signame_events)
         end
         begin
           loop do
-            idx = ipc.wait_any(events, Windows::Synchronize::INFINITE)
-            if idx > 0 && idx <= events.length
-              $log.debug("Got Win32 event \"#{events[idx - 1].name}\"")
+            ipc_idx = ipc.wait_any(events.map {|e| e[:win32_event]}, Windows::Synchronize::INFINITE)
+            event_idx = ipc_idx - 1
+
+            if event_idx >= 0 && event_idx < events.length
+              $log.debug("Got Win32 event \"#{events[event_idx][:win32_event].name}\"")
             else
-              $log.warn("Unexpected reutrn value of Win32::Ipc#wait_any: #{idx}")
+              $log.warn("Unexpected return value of Win32::Ipc#wait_any: #{ipc_idx}")
             end
-            case idx
-            when 2, 6
+            case events[event_idx][:action]
+            when :stop
               stop(true)
-            when 3, 7
+            when :hup
               supervisor_sighup_handler
-            when 4, 8
+            when :usr1
               supervisor_sigusr1_handler
-            when 5, 9
+            when :usr2
               supervisor_sigusr2_handler
-            when 1
+            when :cont
+              supervisor_dump_handler_for_windows
+            when :stop_event_thread
               break
             end
           end
         ensure
-          events.each { |event| event.close }
+          events.each { |event| event[:win32_event].close }
         end
       end
     end
@@ -302,6 +309,26 @@ module Fluent
       $log.error "Failed to reload config file: #{e}"
     end
 
+    def supervisor_dump_handler_for_windows
+      # As for UNIX-like, SIGCONT signal to each process makes the process output its dump-file,
+      # and it is implemented before the implementation of the function for Windows.
+      # It is possible to trap SIGCONT and handle it here also on UNIX-like,
+      # but for backward compatibility, this handler is currently for a Windows-only.
+      raise "[BUG] This function is for Windows ONLY." unless Fluent.windows?
+
+      Thread.new do
+        begin
+          FluentSigdump.dump_windows
+        rescue => e
+          $log.error "failed to dump: #{e}"
+        end
+      end
+
+      send_signal_to_workers(:CONT)
+    rescue => e
+      $log.error "failed to dump: #{e}"
+    end
+
     def kill_worker
       if config[:worker_pid]
         pids = config[:worker_pid].clone
@@ -358,6 +385,14 @@ module Fluent
         restart(true)
       when :USR2
         reload
+      when :CONT
+        dump_all_windows_workers
+      end
+    end
+
+    def dump_all_windows_workers
+      @monitors.each do |m|
+        m.send_command("DUMP\n")
       end
     end
   end
@@ -397,6 +432,7 @@ module Fluent
       log_path = params['log_path']
       chuser = params['chuser']
       chgroup = params['chgroup']
+      chumask = params['chumask']
       log_rotate_age = params['log_rotate_age']
       log_rotate_size = params['log_rotate_size']
 
@@ -436,7 +472,7 @@ module Fluent
         logger_initializer: logger_initializer,
         chuser: chuser,
         chgroup: chgroup,
-        chumask: 0,
+        chumask: chumask,
         suppress_repeated_stacktrace: suppress_repeated_stacktrace,
         ignore_repeated_log_interval: ignore_repeated_log_interval,
         ignore_same_log_interval: ignore_same_log_interval,
@@ -457,7 +493,8 @@ module Fluent
         config_path: path,
         main_cmd: params['main_cmd'],
         signame: params['signame'],
-        disable_shared_socket: params['disable_shared_socket']
+        disable_shared_socket: params['disable_shared_socket'],
+        restart_worker_interval: params['restart_worker_interval'],
       }
       if daemonize
         se_config[:pid_path] = pid_path
@@ -565,6 +602,7 @@ module Fluent
         setup_path: nil,
         chuser: nil,
         chgroup: nil,
+        chumask: "0",
         root_dir: nil,
         suppress_interval: 0,
         suppress_repeated_stacktrace: true,
@@ -578,7 +616,8 @@ module Fluent
         standalone_worker: false,
         signame: nil,
         conf_encoding: 'utf-8',
-        disable_shared_socket: nil
+        disable_shared_socket: nil,
+        config_file_type: :guess,
       }
     end
 
@@ -591,6 +630,7 @@ module Fluent
     end
 
     def initialize(opt)
+      @config_file_type = opt[:config_file_type]
       @daemonize = opt[:daemonize]
       @standalone_worker= opt[:standalone_worker]
       @config_path = opt[:config_path]
@@ -603,6 +643,7 @@ module Fluent
       @plugin_dirs = opt[:plugin_dirs]
       @chgroup = opt[:chgroup]
       @chuser = opt[:chuser]
+      @chumask = opt[:chumask]
 
       @log_rotate_age = opt[:log_rotate_age]
       @log_rotate_size = opt[:log_rotate_size]
@@ -615,7 +656,9 @@ module Fluent
         @conf = Fluent::Config.build(config_path: @config_path,
                                      encoding: @conf_encoding ? @conf_encoding : 'utf-8',
                                      additional_config: @inline_config ? @inline_config : nil,
-                                     use_v1_config: !!@use_v1_config)
+                                     use_v1_config: !!@use_v1_config,
+                                     type: @config_file_type,
+                                    )
         @system_config = build_system_config(@conf)
         if @system_config.log
           @log_rotate_age ||= @system_config.log.rotate_age
@@ -709,7 +752,7 @@ module Fluent
         create_socket_manager if @standalone_worker
         if @standalone_worker
           ServerEngine::Privilege.change(@chuser, @chgroup)
-          File.umask(0)
+          File.umask(@chumask.to_i(8))
         end
         MessagePackFactory.init(enable_time_support: @system_config.enable_msgpack_time_support)
         Fluent::Engine.init(@system_config)
@@ -741,7 +784,13 @@ module Fluent
         $log.warn('the value "-" for `inline_config` is deprecated. See https://github.com/fluent/fluentd/issues/2711')
         @inline_config = STDIN.read
       end
-      @conf = Fluent::Config.build(config_path: @config_path, encoding: @conf_encoding, additional_config: @inline_config, use_v1_config: @use_v1_config)
+      @conf = Fluent::Config.build(
+        config_path: @config_path,
+        encoding: @conf_encoding,
+        additional_config: @inline_config,
+        use_v1_config: @use_v1_config,
+        type: @config_file_type,
+      )
       @system_config = build_system_config(@conf)
 
       @log.level = @system_config.log_level
@@ -819,13 +868,18 @@ module Fluent
         'counter_server' => @system_config.counter_server,
         'log_format' => @system_config.log.format,
         'log_time_format' => @system_config.log.time_format,
-        'disable_shared_socket' => @system_config.disable_shared_socket
+        'disable_shared_socket' => @system_config.disable_shared_socket,
+        'restart_worker_interval' => @system_config.restart_worker_interval,
       }
 
       se = ServerEngine.create(ServerModule, WorkerModule){
         Fluent::Supervisor.load_config(@config_path, params)
       }
-      se.run
+
+      Dir.mktmpdir("fluentd-lock-") do |lock_dir|
+        ENV['FLUENTD_LOCK_DIR'] = lock_dir
+        se.run
+      end
     end
 
     def install_main_process_signal_handlers
@@ -893,6 +947,9 @@ module Fluent
           when "RELOAD"
             $log.debug "fluentd main process get #{cmd} command"
             reload_config
+          when "DUMP"
+            $log.debug "fluentd main process get #{cmd} command"
+            dump
           else
             $log.warn "fluentd main process get unknown command [#{cmd}]"
           end
@@ -926,6 +983,7 @@ module Fluent
             encoding: @conf_encoding,
             additional_config: @inline_config,
             use_v1_config: @use_v1_config,
+            type: @config_file_type,
           )
 
           Fluent::VariableStore.try_to_reset do
@@ -942,6 +1000,16 @@ module Fluent
       end
     end
 
+    def dump
+      Thread.new do
+        begin
+          FluentSigdump.dump_windows
+        rescue => e
+          $log.error("failed to dump: #{e}")
+        end
+      end
+    end
+
     def logging_with_console_output
       yield $log
       unless @log.stdout?
@@ -1051,4 +1119,27 @@ module Fluent
       fluentd_spawn_cmd
     end
   end
+
+  module FluentSigdump
+    def self.dump_windows
+      raise "[BUG] WindowsSigdump::dump is for Windows ONLY." unless Fluent.windows?
+
+      # Sigdump outputs under `/tmp` dir without `SIGDUMP_PATH` specified,
+      # but `/tmp` dir may not exist on Windows by default.
+      # So use the systemroot-temp-dir instead.
+      dump_filepath = ENV['SIGDUMP_PATH'].nil? || ENV['SIGDUMP_PATH'].empty? \
+        ? "#{ENV['windir']}/Temp/fluentd-sigdump-#{Process.pid}.log"
+        : get_path_with_pid(ENV['SIGDUMP_PATH'])
+
+      require 'sigdump'
+      Sigdump.dump(dump_filepath)
+
+      $log.info "dump to #{dump_filepath}."
+    end
+
+    def self.get_path_with_pid(raw_path)
+      path = Pathname.new(raw_path)
+      path.sub_ext("-#{Process.pid}#{path.extname}").to_s
+    end
+  end
 end

data/lib/fluent/system_config.rb

@@ -22,7 +22,7 @@ module Fluent
     include Configurable
 
     SYSTEM_CONFIG_PARAMETERS = [
-      :workers, :root_dir, :log_level,
+      :workers, :restart_worker_interval, :root_dir, :log_level,
       :suppress_repeated_stacktrace, :emit_error_log_interval, :suppress_config_dump,
       :log_event_verbose, :ignore_repeated_log_interval, :ignore_same_log_interval,
       :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
@@ -32,6 +32,7 @@ module Fluent
     ]
 
     config_param :workers,   :integer, default: 1
+    config_param :restart_worker_interval, :time, default: 0
     config_param :root_dir,  :string, default: nil
     config_param :log_level, :enum, list: [:trace, :debug, :info, :warn, :error, :fatal], default: 'info'
     config_param :suppress_repeated_stacktrace, :bool, default: nil

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.14.5'
+  VERSION = '1.15.1'
 
 end

data/lib/fluent/winsvc.rb

@@ -84,6 +84,8 @@ begin
         set_event("#{@service_name}_HUP")
       when 129
         set_event("#{@service_name}_USR1")
+      when 130
+        set_event("#{@service_name}_CONT")
       end
     end
 

data/test/command/test_ctl.rb

@@ -7,7 +7,6 @@ require 'fluent/command/ctl'
 
 class TestFluentdCtl < ::Test::Unit::TestCase
   def assert_win32_event(event_name, command, pid_or_svcname)
-    command, event_suffix = data
     event = Win32::Event.new(event_name)
     ipc = Win32::Ipc.new(event.handle)
     ret = Win32::Ipc::TIMEOUT

data/test/command/test_fluentd.rb

@@ -589,6 +589,39 @@ CONF
       )
     end
 
+    sub_test_case "YAML config format" do
+      test 'success to start the number of workers specified in configuration' do
+        conf = <<'CONF'
+        system:
+          workers: 2
+          root_dir: "#{@root_path}"
+        config:
+          - source:
+              $type: dummy
+              $id: !fluent/s "dummy.#{worker_id}" # check worker_id works or not with actual command
+              $label: '@dummydata'
+              tag: dummy
+              dummy: !fluent/json {"message": !fluent/s "yay from #{hostname}!"}
+
+          - label:
+              $name: '@dummydata'
+              config:
+               - match:
+                  $tag: dummy
+                  $type: "null"
+                  $id: blackhole
+CONF
+        conf_path = create_conf_file('workers1.yaml', conf)
+        assert Dir.exist?(@root_path)
+
+        assert_log_matches(
+          create_cmdline(conf_path),
+          "#0 fluentd worker is now running worker=0",
+          "#1 fluentd worker is now running worker=1"
+        )
+      end
+    end
+
     test 'success to start the number of workers specified by command line option' do
       conf = <<CONF
 <system>

data/test/config/test_system_config.rb

@@ -19,6 +19,7 @@ module Fluent::Config
       @system_config = nil
       @cl_opt = {
         wokers: nil,
+        restart_worker_interval: nil,
         root_dir: nil,
         log: FakeLoggerInitializer.new,
         log_level: Fluent::Log::LEVEL_INFO,
@@ -72,6 +73,7 @@ module Fluent::Config
       sc = Fluent::SystemConfig.new(conf)
       sc.overwrite_variables(**s.for_system_config)
       assert_equal(1, sc.workers)
+      assert_equal(0, sc.restart_worker_interval)
       assert_nil(sc.root_dir)
       assert_equal(Fluent::Log::LEVEL_INFO, sc.log_level)
       assert_nil(sc.suppress_repeated_stacktrace)
@@ -88,6 +90,7 @@ module Fluent::Config
 
     data(
       'workers' => ['workers', 3],
+      'restart_worker_interval' => ['restart_worker_interval', 60],
       'root_dir' => ['root_dir', File.join(TMP_DIR, 'root')],
       'log_level' => ['log_level', 'error'],
       'suppress_repeated_stacktrace' => ['suppress_repeated_stacktrace', true],
@@ -174,7 +177,6 @@ module Fluent::Config
             </log>
           </system>
         EOS
-        s = FakeSupervisor.new
         sc = Fluent::SystemConfig.new(conf)
         assert_equal(3, sc.log.rotate_age)
       end

data/test/config/test_types.rb

@@ -190,7 +190,7 @@ class TestConfigTypes < ::Test::Unit::TestCase
     data("val" => [:val, 'val'],
          "v" => [:v, 'v'],
          "value" => [:value, 'value'])
-    test 'enum' do |(expected, val, list)|
+    test 'enum' do |(expected, val)|
       assert_equal expected, Config::ENUM_TYPE.call(val, {list: [:val, :value, :v]})
     end
 

data/test/plugin/in_tail/test_io_handler.rb

@@ -21,12 +21,20 @@ class IntailIOHandlerTest < Test::Unit::TestCase
     @file.unlink rescue nil
   end
 
+  def create_target_info
+    Fluent::Plugin::TailInput::TargetInfo.new(@file.path, Fluent::FileWrapper.stat(@file.path).ino)
+  end
+
+  def create_watcher
+    Fluent::Plugin::TailInput::TailWatcher.new(create_target_info, nil, nil, nil, nil, nil, nil, nil, nil)
+  end
+
   test '#on_notify load file content and passed it to receive_lines method' do
     text = "this line is test\ntest line is test\n"
     @file.write(text)
     @file.close
 
-    watcher = 'watcher'
+    watcher = create_watcher
 
     update_pos = 0
 
@@ -61,7 +69,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
 
       update_pos = 0
 
-      watcher = 'watcher'
+      watcher = create_watcher
       stub(watcher).pe do
         pe = 'position_file'
         stub(pe).read_pos { 0 }
@@ -92,7 +100,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
 
       update_pos = 0
 
-      watcher = 'watcher'
+      watcher = create_watcher
       stub(watcher).pe do
         pe = 'position_file'
         stub(pe).read_pos { 0 }
@@ -107,6 +115,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
       end
 
       r.on_notify
+      assert_equal text.bytesize, update_pos
       assert_equal 8, returned_lines[0].size
     end
 
@@ -118,7 +127,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
 
       update_pos = 0
 
-      watcher = 'watcher'
+      watcher = create_watcher
       stub(watcher).pe do
         pe = 'position_file'
         stub(pe).read_pos { 0 }
@@ -133,6 +142,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
       end
 
       r.on_notify
+      assert_equal text.bytesize, update_pos
       assert_equal 5, returned_lines[0].size
       assert_equal 3, returned_lines[1].size
     end