fluentd 1.12.1 → 1.12.2

data/lib/fluent/plugin/out_exec_filter.rb

@@ -159,9 +159,9 @@ module Fluent::Plugin
         @added_prefix_string = @add_prefix + '.'
       end
 
-      @respawns = if @child_respawn.nil? or @child_respawn == 'none' or @child_respawn == '0'
+      @respawns = if @child_respawn.nil? || (@child_respawn == 'none') || (@child_respawn == '0')
                     0
-                  elsif @child_respawn == 'inf' or @child_respawn == '-1'
+                  elsif (@child_respawn == 'inf') || (@child_respawn == '-1')
                     -1
                   elsif @child_respawn =~ /^\d+$/
                     @child_respawn.to_i
@@ -251,7 +251,7 @@ module Fluent::Plugin
 
     def tag_remove_prefix(tag)
       if @remove_prefix
-        if (tag[0, @removed_length] == @removed_prefix_string and tag.length > @removed_length) or tag == @removed_prefix_string
+        if ((tag[0, @removed_length] == @removed_prefix_string) && (tag.length > @removed_length)) || (tag == @removed_prefix_string)
           tag = tag[@removed_length..-1] || ''
         end
       end

data/lib/fluent/plugin/out_forward.rb

@@ -166,6 +166,7 @@ module Fluent::Plugin
 
       @usock = nil
       @keep_alive_watcher_interval = 5 # TODO
+      @suspend_flush = false
     end
 
     def configure(conf)
@@ -291,6 +292,15 @@ module Fluent::Plugin
       @require_ack_response
     end
 
+    def overwrite_delayed_commit_timeout
+      # Output#start sets @delayed_commit_timeout by @buffer_config.delayed_commit_timeout
+      # But it should be overwritten by ack_response_timeout to rollback chunks after timeout
+      if @delayed_commit_timeout != @ack_response_timeout
+        log.info "delayed_commit_timeout is overwritten by ack_response_timeout"
+        @delayed_commit_timeout = @ack_response_timeout + 2 # minimum ack_reader IO.select interval is 1s
+      end
+    end
+
     def start
       super
 
@@ -303,13 +313,7 @@ module Fluent::Plugin
       end
 
       if @require_ack_response
-        # Output#start sets @delayed_commit_timeout by @buffer_config.delayed_commit_timeout
-        # But it should be overwritten by ack_response_timeout to rollback chunks after timeout
-        if @delayed_commit_timeout != @ack_response_timeout
-          log.info "delayed_commit_timeout is overwritten by ack_response_timeout"
-          @delayed_commit_timeout = @ack_response_timeout + 2 # minimum ack_reader IO.select interval is 1s
-        end
-
+        overwrite_delayed_commit_timeout
         thread_create(:out_forward_receiving_ack, &method(:ack_reader))
       end
 
@@ -346,6 +350,26 @@ module Fluent::Plugin
       end
     end
 
+    def before_shutdown
+      super
+      @suspend_flush = true
+    end
+
+    def after_shutdown
+      last_ack if @require_ack_response
+      super
+    end
+
+    def try_flush
+      return if @require_ack_response && @suspend_flush
+      super
+    end
+
+    def last_ack
+      overwrite_delayed_commit_timeout
+      ack_check(ack_select_interval)
+    end
+
     def write(chunk)
       return if chunk.empty?
       tag = chunk.metadata.tag
@@ -361,6 +385,7 @@ module Fluent::Plugin
       end
       tag = chunk.metadata.tag
       discovery_manager.select_service { |node| node.send_data(tag, chunk) }
+      last_ack if @require_ack_response && @suspend_flush
     end
 
     def create_transfer_socket(host, port, hostname, &block)
@@ -481,31 +506,39 @@ module Fluent::Plugin
       @connection_manager.purge_obsolete_socks
     end
 
+    def ack_select_interval
+      if @delayed_commit_timeout > 3
+        1
+      else
+        @delayed_commit_timeout / 3.0
+      end
+    end
+
     def ack_reader
-      select_interval = if @delayed_commit_timeout > 3
-                          1
-                        else
-                          @delayed_commit_timeout / 3.0
-                        end
+      select_interval = ack_select_interval
 
       while thread_current_running?
-        @ack_handler.collect_response(select_interval) do |chunk_id, node, sock, result|
-          @connection_manager.close(sock)
-
-          case result
-          when AckHandler::Result::SUCCESS
-            commit_write(chunk_id)
-          when AckHandler::Result::FAILED
-            node.disable!
-            rollback_write(chunk_id, update_retry: false)
-          when AckHandler::Result::CHUNKID_UNMATCHED
-            rollback_write(chunk_id, update_retry: false)
-          else
-            log.warn("BUG: invalid status #{result} #{chunk_id}")
+        ack_check(select_interval)
+      end
+    end
 
-            if chunk_id
-              rollback_write(chunk_id, update_retry: false)
-            end
+    def ack_check(select_interval)
+      @ack_handler.collect_response(select_interval) do |chunk_id, node, sock, result|
+        @connection_manager.close(sock)
+
+        case result
+        when AckHandler::Result::SUCCESS
+          commit_write(chunk_id)
+        when AckHandler::Result::FAILED
+          node.disable!
+          rollback_write(chunk_id, update_retry: false)
+        when AckHandler::Result::CHUNKID_UNMATCHED
+          rollback_write(chunk_id, update_retry: false)
+        else
+          log.warn("BUG: invalid status #{result} #{chunk_id}")
+
+          if chunk_id
+            rollback_write(chunk_id, update_retry: false)
           end
         end
       end

data/lib/fluent/plugin/storage_local.rb

@@ -87,7 +87,7 @@ module Fluent
           if File.exist?(@path)
             raise Fluent::ConfigError, "Plugin storage path '#{@path}' is not readable/writable" unless File.readable?(@path) && File.writable?(@path)
             begin
-              data = open(@path, 'r:utf-8') { |io| io.read }
+              data = File.open(@path, 'r:utf-8') { |io| io.read }
               if data.empty?
                 log.warn "detect empty plugin storage file during startup. Ignored: #{@path}"
                 return
@@ -115,7 +115,7 @@ module Fluent
         return if @on_memory
         return unless File.exist?(@path)
         begin
-          json_string = open(@path, 'r:utf-8'){ |io| io.read }
+          json_string = File.open(@path, 'r:utf-8'){ |io| io.read }
           json = Yajl::Parser.parse(json_string)
           unless json.is_a?(Hash)
             log.error "broken content for plugin storage (Hash required: ignored)", type: json.class
@@ -133,7 +133,7 @@ module Fluent
         tmp_path = @path + '.tmp'
         begin
           json_string = Yajl::Encoder.encode(@store, pretty: @pretty_print)
-          open(tmp_path, 'w:utf-8', @mode) { |io| io.write json_string; io.fsync }
+          File.open(tmp_path, 'w:utf-8', @mode) { |io| io.write json_string; io.fsync }
           File.rename(tmp_path, @path)
         rescue => e
           log.error "failed to save data for plugin storage to file", path: @path, tmp: tmp_path, error: e

data/lib/fluent/supervisor.rb

@@ -385,7 +385,7 @@ module Fluent
       config_mtime = File.mtime(path)
 
       # reuse previous config if last load time is within 5 seconds and mtime of the config file is not changed
-      if Time.now - Time.at(pre_loadtime) < 5 and config_mtime == pre_config_mtime
+      if (Time.now - Time.at(pre_loadtime) < 5) && (config_mtime == pre_config_mtime)
         return params['pre_conf']
       end
 

data/lib/fluent/time.rb

@@ -132,13 +132,14 @@ module Fluent
   end
 
   module TimeMixin
-    TIME_TYPES = ['string', 'unixtime', 'float']
+    TIME_TYPES = ['string', 'unixtime', 'float', 'mixed']
 
     TIME_PARAMETERS = [
       [:time_format, :string, {default: nil}],
       [:localtime, :bool, {default: true}],  # UTC if :localtime is false and :timezone is nil
       [:utc,       :bool, {default: false}], # to turn :localtime false
       [:timezone, :string, {default: nil}],
+      [:time_format_fallbacks, :array, {default: []}], # try time_format, then try fallbacks
     ]
     TIME_FULL_PARAMETERS = [
       # To avoid to define :time_type twice (in plugin_helper/inject)
@@ -170,6 +171,12 @@ module Fluent
           raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
         end
 
+        if conf.has_key?('time_type') and @time_type == :mixed
+          if @time_format.nil? and @time_format_fallbacks.empty?
+            raise Fluent::ConfigError, "time_type is :mixed but time_format and time_format_fallbacks is empty."
+          end
+        end
+
         Fluent::Timezone.validate!(@timezone) if @timezone
       end
     end
@@ -180,6 +187,7 @@ module Fluent
       end
 
       def time_parser_create(type: @time_type, format: @time_format, timezone: @timezone, force_localtime: false)
+        return MixedTimeParser.new(type, format, @localtime, timezone, @utc, force_localtime, @time_format_fallbacks) if type == :mixed
         return NumericTimeParser.new(type) if type != :string
         return TimeParser.new(format, true, nil) if force_localtime
 
@@ -452,4 +460,52 @@ module Fluent
       end
     end
   end
+
+  # MixedTimeParser is available when time_type is set to :mixed
+  #
+  # Use Case 1: primary format is specified explicitly in time_format
+  #  time_type mixed
+  #  time_format %iso8601
+  #  time_format_fallbacks unixtime
+  # Use Case 2: time_format is omitted
+  #  time_type mixed
+  #  time_format_fallbacks %iso8601, unixtime
+  #
+  class MixedTimeParser < TimeParser # to include TimeParseError
+    def initialize(type, format = nil, localtime = nil, timezone = nil, utc = nil, force_localtime = nil, fallbacks = [])
+      @parsers = []
+      fallbacks.unshift(format).each do |fallback|
+        next unless fallback
+        case fallback
+        when 'unixtime', 'float'
+          @parsers << NumericTimeParser.new(fallback, localtime, timezone)
+        else
+          if force_localtime
+            @parsers << TimeParser.new(fallback, true, nil)
+          else
+            localtime = localtime && (timezone.nil? && !utc)
+            @parsers << TimeParser.new(fallback, localtime, timezone)
+          end
+        end
+      end
+    end
+
+    def parse(value)
+      @parsers.each do |parser|
+        begin
+          Float(value) if parser.class == Fluent::NumericTimeParser
+        rescue
+          next
+        end
+        begin
+          return parser.parse(value)
+        rescue
+          # skip TimeParseError
+        end
+      end
+      fallback_class = @parsers.collect do |parser| parser.class end.join(",")
+      raise TimeParseError, "invalid time format: value = #{value}, even though fallbacks: #{fallback_class}"
+    end
+  end
+
 end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.12.1'
+  VERSION = '1.12.2'
 
 end

data/test/command/test_fluentd.rb

@@ -878,6 +878,8 @@ CONF
     end
 
     test "without RUBYOPT" do
+      saved_ruby_opt = ENV["RUBYOPT"]
+      ENV["RUBYOPT"] = nil
       conf = <<CONF
 <source>
   @type dummy
@@ -889,6 +891,8 @@ CONF
 CONF
       conf_path = create_conf_file('rubyopt_test.conf', conf)
       assert_log_matches(create_cmdline(conf_path), '-Eascii-8bit:ascii-8bit')
+    ensure
+      ENV["RUBYOPT"] = saved_ruby_opt
     end
 
     test 'invalid values are set to RUBYOPT' do
@@ -912,6 +916,8 @@ CONF
 
     # https://github.com/fluent/fluentd/issues/2915
     test "ruby path contains spaces" do
+      saved_ruby_opt = ENV["RUBYOPT"]
+      ENV["RUBYOPT"] = nil
       conf = <<CONF
 <source>
   @type dummy
@@ -940,6 +946,8 @@ CONF
         'spawn command to main:',
         '-Eascii-8bit:ascii-8bit'
       )
+    ensure
+      ENV["RUBYOPT"] = saved_ruby_opt
     end
 
     test 'success to start workers when file buffer is configured in non-workers way only for specific worker' do

data/test/config/test_configurable.rb

@@ -1453,7 +1453,7 @@ module Fluent::Config
         @example = ConfigurableSpec::ExampleWithSkipAccessor.new
         @example.configure(config_element('ROOT'))
         assert_equal 'example7', @example.instance_variable_get(:@name)
-        assert_raise NoMethodError.new("undefined method `name' for #{@example}") do
+        assert_raise NoMethodError do
           @example.name
         end
       end

data/test/plugin/in_tail/test_position_file.rb

@@ -147,7 +147,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
       write_data(@file, TEST_CONTENT)
       pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, **{logger: $log})
 
-      valid_target_info = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', Fluent::FileWrapper.stat(@file).ino)
+      valid_target_info = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', File.stat(@file).ino)
       f = pf[valid_target_info]
       assert_equal Fluent::Plugin::TailInput::FilePositionEntry, f.class
       assert_equal 2, f.read_pos
@@ -177,7 +177,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
       assert_equal 0, f.read_inode
       assert_equal 0, f.read_pos
 
-      pf[Fluent::Plugin::TailInput::TargetInfo.new('valid_path', Fluent::FileWrapper.stat(@file).ino)].update(1, 2)
+      pf[Fluent::Plugin::TailInput::TargetInfo.new('valid_path', File.stat(@file).ino)].update(1, 2)
 
       f = pf[Fluent::Plugin::TailInput::TargetInfo.new('nonexist_path', -1)]
       assert_equal 0, f.read_inode
@@ -193,7 +193,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
     test 'deletes entry by path' do
       write_data(@file, TEST_CONTENT)
       pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, logger: $log)
-      inode1 = Fluent::FileWrapper.stat(@file).ino
+      inode1 = File.stat(@file).ino
       target_info1 = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', inode1)
       p1 = pf[target_info1]
       assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p1.class
@@ -201,7 +201,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
       pf.unwatch(target_info1)
       assert_equal p1.read_pos, Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION
 
-      inode2 = Fluent::FileWrapper.stat(@file).ino
+      inode2 = File.stat(@file).ino
       target_info2 = Fluent::Plugin::TailInput::TargetInfo.new('valid_path', inode2)
       p2 = pf[target_info2]
       assert_equal Fluent::Plugin::TailInput::FilePositionEntry, p2.class

data/test/plugin/out_forward/test_connection_manager.rb

@@ -81,6 +81,8 @@ class ConnectionManager < Test::Unit::TestCase
 
     sub_test_case 'when socket_cache exists' do
       test 'calls connect_keepalive' do
+        omit "Proxy of RR doesn't support kwargs of Ruby 3 yet" if RUBY_VERSION.split('.')[0].to_i >= 3
+
         cache = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
         mock(cache).checkin('sock').never
 
@@ -99,6 +101,8 @@ class ConnectionManager < Test::Unit::TestCase
       end
 
       test 'calls connect_keepalive and closes socket with block' do
+        omit "Proxy of RR doesn't support kwargs of Ruby 3 yet" if RUBY_VERSION.split('.')[0].to_i >= 3
+
         cache = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
         mock(cache).checkin('sock').once
 
@@ -118,6 +122,8 @@ class ConnectionManager < Test::Unit::TestCase
       end
 
       test 'does not call dec_ref when ack is passed' do
+        omit "Proxy of RR doesn't support kwargs of Ruby 3 yet" if RUBY_VERSION.split('.')[0].to_i >= 3
+
         cache = Fluent::Plugin::ForwardOutput::SocketCache.new(10, $log)
         mock(cache).checkin('sock').never
         sock = 'sock'

data/test/plugin/test_in_exec.rb

@@ -255,7 +255,7 @@ EOC
     assert{ d.events.length > 0 }
     d.events.each do |event|
       assert_equal 'test', event[0]
-      assert_match /LoadError/, event[2]['message']
+      assert_match(/LoadError/, event[2]['message'])
     end
   end
 end

data/test/plugin/test_in_tail.rb

@@ -6,6 +6,8 @@ require 'fluent/system_config'
 require 'net/http'
 require 'flexmock/test_unit'
 require 'timecop'
+require 'tmpdir'
+require 'securerandom'
 
 class TailInputTest < Test::Unit::TestCase
   include FlexMock::TestCase
@@ -22,28 +24,64 @@ class TailInputTest < Test::Unit::TestCase
   end
 
   def cleanup_directory(path)
-    begin
-      FileUtils.rm_f(path, secure: true)
-    rescue ArgumentError
-      FileUtils.rm_f(path) # For Ruby 2.6 or before.
+    unless Dir.exist?(path)
+      FileUtils.mkdir_p(path)
+      return
     end
-    if File.exist?(path)
-      FileUtils.remove_entry_secure(path, true)
+
+    if Fluent.windows?
+      Dir.glob("*", base: path).each do |name|
+        begin
+          cleanup_file(File.join(path, name))
+        rescue
+          # expect test driver block release already owned file handle.
+        end
+      end
+    else
+      begin
+        FileUtils.rm_f(path, secure:true)
+      rescue ArgumentError
+        FileUtils.rm_f(path) # For Ruby 2.6 or before.
+      end
+      if File.exist?(path)
+        FileUtils.remove_entry_secure(path, true)
+      end
     end
     FileUtils.mkdir_p(path)
   end
 
   def cleanup_file(path)
-    begin
-      FileUtils.rm_f(path, secure: true)
-    rescue ArgumentError
-      FileUtils.rm_f(path) # For Ruby 2.6 or before.
-    end
-    if File.exist?(path)
-      # ensure files are closed for Windows, on which deleted files
-      # are still visible from filesystem
-      GC.start(full_mark: true, immediate_mark: true, immediate_sweep: true)
-      FileUtils.remove_entry_secure(path, true)
+    if Fluent.windows?
+      # On Windows, when the file or directory is removed and created
+      # frequently, there is a case that creating file or directory will
+      # fail. This situation is caused by pending file or directory
+      # deletion which is mentioned on win32 API document [1]
+      # As a workaround, execute rename and remove method.
+      #
+      # [1] https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilea#files
+      #
+      file = File.join(Dir.tmpdir, SecureRandom.hex(10))
+      begin
+        FileUtils.mv(path, file)
+        FileUtils.rm_rf(file, secure: true)
+      rescue ArgumentError
+        FileUtils.rm_rf(file) # For Ruby 2.6 or before.
+      end
+      if File.exist?(file)
+        # ensure files are closed for Windows, on which deleted files
+        # are still visible from filesystem
+        GC.start(full_mark: true, immediate_mark: true, immediate_sweep: true)
+        FileUtils.remove_entry_secure(file, true)
+      end
+    else
+      begin
+        FileUtils.rm_f(path, secure: true)
+      rescue ArgumentError
+        FileUtils.rm_f(path) # For Ruby 2.6 or before.
+      end
+      if File.exist?(path)
+        FileUtils.remove_entry_secure(path, true)
+      end
     end
   end