fluentd 1.14.6 → 1.15.0

data/lib/fluent/plugin/output.rb

@@ -235,6 +235,7 @@ module Fluent
         @dequeued_chunks_mutex = nil
         @output_enqueue_thread = nil
         @output_flush_threads = nil
+        @output_flush_thread_current_position = 0
 
         @simple_chunking = nil
         @chunk_keys = @chunk_key_accessors = @chunk_key_time = @chunk_key_tag = nil
@@ -492,6 +493,7 @@ module Fluent
           @dequeued_chunks = []
           @dequeued_chunks_mutex = Mutex.new
 
+          @output_flush_thread_current_position = 0
           @buffer_config.flush_thread_count.times do |i|
             thread_title = "flush_thread_#{i}".to_sym
             thread_state = FlushThreadState.new(nil, nil, Mutex.new, ConditionVariable.new)
@@ -503,7 +505,6 @@ module Fluent
               @output_flush_threads << thread_state
             end
           end
-          @output_flush_thread_current_position = 0
 
           if !@under_plugin_development && (@flush_mode == :interval || @chunk_key_time)
             @output_enqueue_thread = thread_create(:enqueue_thread, &method(:enqueue_thread_run))

data/lib/fluent/plugin/parser_syslog.rb

@@ -484,7 +484,7 @@ module Fluent
 
         time = begin
                  @time_parser_rfc5424.parse(time_str)
-               rescue Fluent::TimeParser::TimeParseError => e
+               rescue Fluent::TimeParser::TimeParseError
                  @time_parser_rfc5424_without_subseconds.parse(time_str)
                end
         record['time'] = time_str if @keep_time_key

data/lib/fluent/plugin_helper/server.rb

@@ -267,7 +267,9 @@ module Fluent
           ### Socket Params ###
 
           # SO_LINGER 0 to send RST rather than FIN to avoid lots of connections sitting in TIME_WAIT at src.
-          # Set positive value if needing to send FIN on closing.
+          # Set positive value if needing to send FIN on closing on non-Windows.
+          # (On Windows, Fluentd can send FIN with zero `linger_timeout` since Fluentd doesn't set 0 to SO_LINGER on Windows.
+          # See `socket_option.rb`.)
           # NOTE:
             # Socket-options can be specified from each plugin as needed, so most of them is not defined here for now.
             # This is because there is no positive reason to do so.

data/lib/fluent/plugin_helper/service_discovery.rb

@@ -44,7 +44,7 @@ module Fluent
 
         @discovery_manager.start
         unless @discovery_manager.static_config?
-          timer_execute(@_plugin_helper_service_discovery_title, @_plugin_helper_service_discovery_iterval) do
+          timer_execute(@_plugin_helper_service_discovery_title, @_plugin_helper_service_discovery_interval) do
             @discovery_manager.run_once
           end
         end
@@ -96,7 +96,7 @@ module Fluent
       # @param custom_build_method [Proc]
       def service_discovery_create_manager(title, configurations:, load_balancer: nil, custom_build_method: nil, interval: 3)
         @_plugin_helper_service_discovery_title = title
-        @_plugin_helper_service_discovery_iterval = interval
+        @_plugin_helper_service_discovery_interval = interval
 
         @discovery_manager = Fluent::PluginHelper::ServiceDiscovery::Manager.new(
           log: log,

data/lib/fluent/supervisor.rb

@@ -16,6 +16,7 @@
 
 require 'fileutils'
 require 'open3'
+require 'pathname'
 
 require 'fluent/config'
 require 'fluent/counter'
@@ -215,44 +216,50 @@ module Fluent
       Thread.new do
         ipc = Win32::Ipc.new(nil)
         events = [
-          Win32::Event.new("#{@pid_signame}_STOP_EVENT_THREAD"),
-          Win32::Event.new("#{@pid_signame}"),
-          Win32::Event.new("#{@pid_signame}_HUP"),
-          Win32::Event.new("#{@pid_signame}_USR1"),
-          Win32::Event.new("#{@pid_signame}_USR2"),
+          {win32_event: Win32::Event.new("#{@pid_signame}_STOP_EVENT_THREAD"), action: :stop_event_thread},
+          {win32_event: Win32::Event.new("#{@pid_signame}"), action: :stop},
+          {win32_event: Win32::Event.new("#{@pid_signame}_HUP"), action: :hup},
+          {win32_event: Win32::Event.new("#{@pid_signame}_USR1"), action: :usr1},
+          {win32_event: Win32::Event.new("#{@pid_signame}_USR2"), action: :usr2},
+          {win32_event: Win32::Event.new("#{@pid_signame}_CONT"), action: :cont},
         ]
         if @signame
           signame_events = [
-            Win32::Event.new("#{@signame}"),
-            Win32::Event.new("#{@signame}_HUP"),
-            Win32::Event.new("#{@signame}_USR1"),
-            Win32::Event.new("#{@signame}_USR2"),
+            {win32_event: Win32::Event.new("#{@signame}"), action: :stop},
+            {win32_event: Win32::Event.new("#{@signame}_HUP"), action: :hup},
+            {win32_event: Win32::Event.new("#{@signame}_USR1"), action: :usr1},
+            {win32_event: Win32::Event.new("#{@signame}_USR2"), action: :usr2},
+            {win32_event: Win32::Event.new("#{@signame}_CONT"), action: :cont},
           ]
           events.concat(signame_events)
         end
         begin
           loop do
-            idx = ipc.wait_any(events, Windows::Synchronize::INFINITE)
-            if idx > 0 && idx <= events.length
-              $log.debug("Got Win32 event \"#{events[idx - 1].name}\"")
+            ipc_idx = ipc.wait_any(events.map {|e| e[:win32_event]}, Windows::Synchronize::INFINITE)
+            event_idx = ipc_idx - 1
+
+            if event_idx >= 0 && event_idx < events.length
+              $log.debug("Got Win32 event \"#{events[event_idx][:win32_event].name}\"")
             else
-              $log.warn("Unexpected reutrn value of Win32::Ipc#wait_any: #{idx}")
+              $log.warn("Unexpected return value of Win32::Ipc#wait_any: #{ipc_idx}")
             end
-            case idx
-            when 2, 6
+            case events[event_idx][:action]
+            when :stop
               stop(true)
-            when 3, 7
+            when :hup
               supervisor_sighup_handler
-            when 4, 8
+            when :usr1
               supervisor_sigusr1_handler
-            when 5, 9
+            when :usr2
               supervisor_sigusr2_handler
-            when 1
+            when :cont
+              supervisor_dump_handler_for_windows
+            when :stop_event_thread
               break
             end
           end
         ensure
-          events.each { |event| event.close }
+          events.each { |event| event[:win32_event].close }
         end
       end
     end
@@ -302,6 +309,26 @@ module Fluent
       $log.error "Failed to reload config file: #{e}"
     end
 
+    def supervisor_dump_handler_for_windows
+      # As for UNIX-like, SIGCONT signal to each process makes the process output its dump-file,
+      # and it is implemented before the implementation of the function for Windows.
+      # It is possible to trap SIGCONT and handle it here also on UNIX-like,
+      # but for backward compatibility, this handler is currently for a Windows-only.
+      raise "[BUG] This function is for Windows ONLY." unless Fluent.windows?
+
+      Thread.new do
+        begin
+          FluentSigdump.dump_windows
+        rescue => e
+          $log.error "failed to dump: #{e}"
+        end
+      end
+
+      send_signal_to_workers(:CONT)
+    rescue => e
+      $log.error "failed to dump: #{e}"
+    end
+
     def kill_worker
       if config[:worker_pid]
         pids = config[:worker_pid].clone
@@ -358,6 +385,14 @@ module Fluent
         restart(true)
       when :USR2
         reload
+      when :CONT
+        dump_all_windows_workers
+      end
+    end
+
+    def dump_all_windows_workers
+      @monitors.each do |m|
+        m.send_command("DUMP\n")
       end
     end
   end
@@ -458,7 +493,8 @@ module Fluent
         config_path: path,
         main_cmd: params['main_cmd'],
         signame: params['signame'],
-        disable_shared_socket: params['disable_shared_socket']
+        disable_shared_socket: params['disable_shared_socket'],
+        restart_worker_interval: params['restart_worker_interval'],
       }
       if daemonize
         se_config[:pid_path] = pid_path
@@ -580,7 +616,8 @@ module Fluent
         standalone_worker: false,
         signame: nil,
         conf_encoding: 'utf-8',
-        disable_shared_socket: nil
+        disable_shared_socket: nil,
+        config_file_type: :guess,
       }
     end
 
@@ -593,6 +630,7 @@ module Fluent
     end
 
     def initialize(opt)
+      @config_file_type = opt[:config_file_type]
       @daemonize = opt[:daemonize]
       @standalone_worker= opt[:standalone_worker]
       @config_path = opt[:config_path]
@@ -618,7 +656,9 @@ module Fluent
         @conf = Fluent::Config.build(config_path: @config_path,
                                      encoding: @conf_encoding ? @conf_encoding : 'utf-8',
                                      additional_config: @inline_config ? @inline_config : nil,
-                                     use_v1_config: !!@use_v1_config)
+                                     use_v1_config: !!@use_v1_config,
+                                     type: @config_file_type,
+                                    )
         @system_config = build_system_config(@conf)
         if @system_config.log
           @log_rotate_age ||= @system_config.log.rotate_age
@@ -744,7 +784,13 @@ module Fluent
         $log.warn('the value "-" for `inline_config` is deprecated. See https://github.com/fluent/fluentd/issues/2711')
         @inline_config = STDIN.read
       end
-      @conf = Fluent::Config.build(config_path: @config_path, encoding: @conf_encoding, additional_config: @inline_config, use_v1_config: @use_v1_config)
+      @conf = Fluent::Config.build(
+        config_path: @config_path,
+        encoding: @conf_encoding,
+        additional_config: @inline_config,
+        use_v1_config: @use_v1_config,
+        type: @config_file_type,
+      )
       @system_config = build_system_config(@conf)
 
       @log.level = @system_config.log_level
@@ -822,7 +868,8 @@ module Fluent
         'counter_server' => @system_config.counter_server,
         'log_format' => @system_config.log.format,
         'log_time_format' => @system_config.log.time_format,
-        'disable_shared_socket' => @system_config.disable_shared_socket
+        'disable_shared_socket' => @system_config.disable_shared_socket,
+        'restart_worker_interval' => @system_config.restart_worker_interval,
       }
 
       se = ServerEngine.create(ServerModule, WorkerModule){
@@ -896,6 +943,9 @@ module Fluent
           when "RELOAD"
             $log.debug "fluentd main process get #{cmd} command"
             reload_config
+          when "DUMP"
+            $log.debug "fluentd main process get #{cmd} command"
+            dump
           else
             $log.warn "fluentd main process get unknown command [#{cmd}]"
           end
@@ -929,6 +979,7 @@ module Fluent
             encoding: @conf_encoding,
             additional_config: @inline_config,
             use_v1_config: @use_v1_config,
+            type: @config_file_type,
           )
 
           Fluent::VariableStore.try_to_reset do
@@ -945,6 +996,16 @@ module Fluent
       end
     end
 
+    def dump
+      Thread.new do
+        begin
+          FluentSigdump.dump_windows
+        rescue => e
+          $log.error("failed to dump: #{e}")
+        end
+      end
+    end
+
     def logging_with_console_output
       yield $log
       unless @log.stdout?
@@ -1054,4 +1115,27 @@ module Fluent
       fluentd_spawn_cmd
     end
   end
+
+  module FluentSigdump
+    def self.dump_windows
+      raise "[BUG] WindowsSigdump::dump is for Windows ONLY." unless Fluent.windows?
+
+      # Sigdump outputs under `/tmp` dir without `SIGDUMP_PATH` specified,
+      # but `/tmp` dir may not exist on Windows by default.
+      # So use the systemroot-temp-dir instead.
+      dump_filepath = ENV['SIGDUMP_PATH'].nil? || ENV['SIGDUMP_PATH'].empty? \
+        ? "#{ENV['windir']}/Temp/fluentd-sigdump-#{Process.pid}.log"
+        : get_path_with_pid(ENV['SIGDUMP_PATH'])
+
+      require 'sigdump'
+      Sigdump.dump(dump_filepath)
+
+      $log.info "dump to #{dump_filepath}."
+    end
+
+    def self.get_path_with_pid(raw_path)
+      path = Pathname.new(raw_path)
+      path.sub_ext("-#{Process.pid}#{path.extname}").to_s
+    end
+  end
 end

data/lib/fluent/system_config.rb

@@ -22,7 +22,7 @@ module Fluent
     include Configurable
 
     SYSTEM_CONFIG_PARAMETERS = [
-      :workers, :root_dir, :log_level,
+      :workers, :restart_worker_interval, :root_dir, :log_level,
       :suppress_repeated_stacktrace, :emit_error_log_interval, :suppress_config_dump,
       :log_event_verbose, :ignore_repeated_log_interval, :ignore_same_log_interval,
       :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
@@ -32,6 +32,7 @@ module Fluent
     ]
 
     config_param :workers,   :integer, default: 1
+    config_param :restart_worker_interval, :time, default: 0
     config_param :root_dir,  :string, default: nil
     config_param :log_level, :enum, list: [:trace, :debug, :info, :warn, :error, :fatal], default: 'info'
     config_param :suppress_repeated_stacktrace, :bool, default: nil

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.14.6'
+  VERSION = '1.15.0'
 
 end

data/lib/fluent/winsvc.rb

@@ -84,6 +84,8 @@ begin
         set_event("#{@service_name}_HUP")
       when 129
         set_event("#{@service_name}_USR1")
+      when 130
+        set_event("#{@service_name}_CONT")
       end
     end
 

data/test/command/test_ctl.rb

@@ -7,7 +7,6 @@ require 'fluent/command/ctl'
 
 class TestFluentdCtl < ::Test::Unit::TestCase
   def assert_win32_event(event_name, command, pid_or_svcname)
-    command, event_suffix = data
     event = Win32::Event.new(event_name)
     ipc = Win32::Ipc.new(event.handle)
     ret = Win32::Ipc::TIMEOUT

data/test/command/test_fluentd.rb

@@ -589,6 +589,39 @@ CONF
       )
     end
 
+    sub_test_case "YAML config format" do
+      test 'success to start the number of workers specified in configuration' do
+        conf = <<'CONF'
+        system:
+          workers: 2
+          root_dir: "#{@root_path}"
+        config:
+          - source:
+              $type: dummy
+              $id: !fluent/s "dummy.#{worker_id}" # check worker_id works or not with actual command
+              $label: '@dummydata'
+              tag: dummy
+              dummy: !fluent/json {"message": !fluent/s "yay from #{hostname}!"}
+
+          - label:
+              $name: '@dummydata'
+              config:
+               - match:
+                  $tag: dummy
+                  $type: "null"
+                  $id: blackhole
+CONF
+        conf_path = create_conf_file('workers1.yaml', conf)
+        assert Dir.exist?(@root_path)
+
+        assert_log_matches(
+          create_cmdline(conf_path),
+          "#0 fluentd worker is now running worker=0",
+          "#1 fluentd worker is now running worker=1"
+        )
+      end
+    end
+
     test 'success to start the number of workers specified by command line option' do
       conf = <<CONF
 <system>

data/test/config/test_system_config.rb

@@ -19,6 +19,7 @@ module Fluent::Config
       @system_config = nil
       @cl_opt = {
         wokers: nil,
+        restart_worker_interval: nil,
         root_dir: nil,
         log: FakeLoggerInitializer.new,
         log_level: Fluent::Log::LEVEL_INFO,
@@ -72,6 +73,7 @@ module Fluent::Config
       sc = Fluent::SystemConfig.new(conf)
       sc.overwrite_variables(**s.for_system_config)
       assert_equal(1, sc.workers)
+      assert_equal(0, sc.restart_worker_interval)
       assert_nil(sc.root_dir)
       assert_equal(Fluent::Log::LEVEL_INFO, sc.log_level)
       assert_nil(sc.suppress_repeated_stacktrace)
@@ -88,6 +90,7 @@ module Fluent::Config
 
     data(
       'workers' => ['workers', 3],
+      'restart_worker_interval' => ['restart_worker_interval', 60],
       'root_dir' => ['root_dir', File.join(TMP_DIR, 'root')],
       'log_level' => ['log_level', 'error'],
       'suppress_repeated_stacktrace' => ['suppress_repeated_stacktrace', true],
@@ -174,7 +177,6 @@ module Fluent::Config
             </log>
           </system>
         EOS
-        s = FakeSupervisor.new
         sc = Fluent::SystemConfig.new(conf)
         assert_equal(3, sc.log.rotate_age)
       end

data/test/config/test_types.rb

@@ -190,7 +190,7 @@ class TestConfigTypes < ::Test::Unit::TestCase
     data("val" => [:val, 'val'],
          "v" => [:v, 'v'],
          "value" => [:value, 'value'])
-    test 'enum' do |(expected, val, list)|
+    test 'enum' do |(expected, val)|
       assert_equal expected, Config::ENUM_TYPE.call(val, {list: [:val, :value, :v]})
     end
 

data/test/plugin/in_tail/test_io_handler.rb

@@ -21,12 +21,20 @@ class IntailIOHandlerTest < Test::Unit::TestCase
     @file.unlink rescue nil
   end
 
+  def create_target_info
+    Fluent::Plugin::TailInput::TargetInfo.new(@file.path, Fluent::FileWrapper.stat(@file.path).ino)
+  end
+
+  def create_watcher
+    Fluent::Plugin::TailInput::TailWatcher.new(create_target_info, nil, nil, nil, nil, nil, nil, nil, nil)
+  end
+
   test '#on_notify load file content and passed it to receive_lines method' do
     text = "this line is test\ntest line is test\n"
     @file.write(text)
     @file.close
 
-    watcher = 'watcher'
+    watcher = create_watcher
 
     update_pos = 0
 
@@ -61,7 +69,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
 
       update_pos = 0
 
-      watcher = 'watcher'
+      watcher = create_watcher
       stub(watcher).pe do
         pe = 'position_file'
         stub(pe).read_pos { 0 }
@@ -92,7 +100,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
 
       update_pos = 0
 
-      watcher = 'watcher'
+      watcher = create_watcher
       stub(watcher).pe do
         pe = 'position_file'
         stub(pe).read_pos { 0 }
@@ -107,6 +115,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
       end
 
       r.on_notify
+      assert_equal text.bytesize, update_pos
       assert_equal 8, returned_lines[0].size
     end
 
@@ -118,7 +127,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
 
       update_pos = 0
 
-      watcher = 'watcher'
+      watcher = create_watcher
       stub(watcher).pe do
         pe = 'position_file'
         stub(pe).read_pos { 0 }
@@ -133,6 +142,7 @@ class IntailIOHandlerTest < Test::Unit::TestCase
       end
 
       r.on_notify
+      assert_equal text.bytesize, update_pos
       assert_equal 5, returned_lines[0].size
       assert_equal 3, returned_lines[1].size
     end

data/test/plugin/in_tail/test_position_file.rb

@@ -40,10 +40,6 @@ class IntailPositionFileTest < Test::Unit::TestCase
 
   test '.load' do
     write_data(@file, TEST_CONTENT)
-    paths = {
-      "valid_path" => Fluent::Plugin::TailInput::TargetInfo.new("valid_path", 1),
-      "inode23bit" => Fluent::Plugin::TailInput::TargetInfo.new("inode23bit", 2),
-    }
     Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log})
 
     @file.seek(0)
@@ -146,11 +142,6 @@ class IntailPositionFileTest < Test::Unit::TestCase
   sub_test_case '#load' do
     test 'compact invalid and convert 32 bit inode value' do
       write_data(@file, TEST_CONTENT)
-      invalid_path = "invalidpath100000000000000000000000000000000"
-      paths = TEST_CONTENT_PATHS.merge({
-        invalid_path => Fluent::Plugin::TailInput::TargetInfo.new(invalid_path, 0),
-        "unwatched" => Fluent::Plugin::TailInput::TargetInfo.new("unwatched", 0),
-      })
       Fluent::Plugin::TailInput::PositionFile.load(@file, false, TEST_CONTENT_PATHS, **{logger: $log})
 
       @file.seek(0)
@@ -322,58 +313,4 @@ class IntailPositionFileTest < Test::Unit::TestCase
       assert_equal 2, f.read_inode
     end
   end
-
-  sub_test_case "TargetInfo equality rules" do
-    sub_test_case "== operator" do
-      def test_equal
-        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
-        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1235)
-
-        assert_equal t1, t2
-      end
-
-      def test_not_equal
-        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
-        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test2", 1234)
-
-        assert_not_equal t1, t2
-      end
-    end
-
-    sub_test_case "eql? method" do
-      def test_eql?
-        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
-        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test", 5321)
-
-        assert do
-          t1.eql? t2
-        end
-      end
-
-      def test_not_eql?
-        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test2", 1234)
-        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test3", 1234)
-
-        assert do
-          !t1.eql? t2
-        end
-      end
-    end
-
-    sub_test_case "hash" do
-      def test_equal
-        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
-        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test", 7321)
-
-        assert_equal t1.hash, t2.hash
-      end
-
-      def test_not_equal
-        t1 = Fluent::Plugin::TailInput::TargetInfo.new("test", 1234)
-        t2 = Fluent::Plugin::TailInput::TargetInfo.new("test2", 1234)
-
-        assert_not_equal t1.hash, t2.hash
-      end
-    end
-  end
 end

data/test/plugin/out_forward/test_socket_cache.rb

@@ -110,6 +110,10 @@ class SocketCacheTest < Test::Unit::TestCase
   end
 
   sub_test_case 'purge_obsolete_socks' do
+    def teardown
+      Timecop.return
+    end
+
     test 'delete key in inactive_socks' do
       c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
       sock = mock!.close { 'closed' }.subject
@@ -134,7 +138,7 @@ class SocketCacheTest < Test::Unit::TestCase
       c.checkin(sock)
 
       # wait timeout
-      Timecop.freeze(Time.parse('2016-04-13 14:20:00 +0900'))
+      Timecop.freeze(Time.parse('2016-04-13 14:00:11 +0900'))
       c.checkout_or('key') { sock2 }
 
       assert_equal(1, c.instance_variable_get(:@inflight_sockets).size)
@@ -145,5 +149,26 @@ class SocketCacheTest < Test::Unit::TestCase
       assert_equal(1, c.instance_variable_get(:@inflight_sockets).size)
       assert_equal(sock2, c.instance_variable_get(:@inflight_sockets).values.first.sock)
     end
+
+    test 'should not purge just after checkin and purge after timeout' do
+      Timecop.freeze(Time.parse('2016-04-13 14:00:00 +0900'))
+
+      c = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
+      sock = dont_allow(mock!).close
+      stub(sock).inspect
+      c.checkout_or('key') { sock }
+
+      Timecop.freeze(Time.parse('2016-04-13 14:00:11 +0900'))
+      c.checkin(sock)
+
+      assert_equal(1, c.instance_variable_get(:@available_sockets).size)
+      c.purge_obsolete_socks
+      assert_equal(1, c.instance_variable_get(:@available_sockets).size)
+
+      Timecop.freeze(Time.parse('2016-04-13 14:00:22 +0900'))
+      assert_equal(1, c.instance_variable_get(:@available_sockets).size)
+      c.purge_obsolete_socks
+      assert_equal(0, c.instance_variable_get(:@available_sockets).size)
+    end
   end
 end