fluentd 1.16.0-x64-mingw32 → 1.16.2-x64-mingw32

data/lib/fluent/plugin/in_tail.rb

@@ -370,17 +370,30 @@ module Fluent::Plugin
     def refresh_watchers
       target_paths_hash = expand_paths
       existence_paths_hash = existence_path
-      
+
       log.debug {
         target_paths_str = target_paths_hash.collect { |key, target_info| target_info.path }.join(",")
         existence_paths_str = existence_paths_hash.collect { |key, target_info| target_info.path }.join(",")
         "tailing paths: target = #{target_paths_str} | existing = #{existence_paths_str}"
       }
 
-      unwatched_hash = existence_paths_hash.reject {|key, value| target_paths_hash.key?(key)}
+      if !@follow_inodes
+        need_unwatch_in_stop_watchers = true
+      else
+        # When using @follow_inodes, need this to unwatch the rotated old inode when it disappears.
+        # After `update_watcher` detaches an old TailWatcher, the inode is lost from the `@tails`.
+        # So that inode can't be contained in `removed_hash`, and can't be unwatched by `stop_watchers`.
+        #
+        # This logic may work for `@follow_inodes false` too.
+        # Just limiting the case to supress the impact to existing logics.
+        @pf&.unwatch_removed_targets(target_paths_hash)
+        need_unwatch_in_stop_watchers = false
+      end
+
+      removed_hash = existence_paths_hash.reject {|key, value| target_paths_hash.key?(key)}
       added_hash = target_paths_hash.reject {|key, value| existence_paths_hash.key?(key)}
 
-      stop_watchers(unwatched_hash, immediate: false, unwatched: true) unless unwatched_hash.empty?
+      stop_watchers(removed_hash, unwatched: need_unwatch_in_stop_watchers) unless removed_hash.empty?
       start_watchers(added_hash) unless added_hash.empty?
       @startup = false if @startup
     end
@@ -484,8 +497,26 @@ module Fluent::Plugin
     end
 
     # refresh_watchers calls @tails.keys so we don't use stop_watcher -> start_watcher sequence for safety.
-    def update_watcher(target_info, pe)
-      path = target_info.path
+    def update_watcher(tail_watcher, pe, new_inode)
+      # TODO we should use another callback for this.
+      # To supress impact to existing logics, limit the case to `@follow_inodes`.
+      # We may not need `@follow_inodes` condition.
+      if @follow_inodes && new_inode.nil?
+        # nil inode means the file disappeared, so we only need to stop it.
+        @tails.delete(tail_watcher.path)
+        # https://github.com/fluent/fluentd/pull/4237#issuecomment-1633358632 
+        # Because of this problem, log duplication can occur during `rotate_wait`.
+        # Need to set `rotate_wait 0` for a workaround.
+        # Duplication will occur if `refresh_watcher` is called during the `rotate_wait`.
+        # In that case, `refresh_watcher` will add the new TailWatcher to tail the same target,
+        # and it causes the log duplication.
+        # (Other `detach_watcher_after_rotate_wait` may have the same problem.
+        #  We need the mechanism not to add duplicated TailWathcer with detaching TailWatcher.)
+        detach_watcher_after_rotate_wait(tail_watcher, pe.read_inode)
+        return
+      end
+
+      path = tail_watcher.path
 
       log.info("detected rotation of #{path}; waiting #{@rotate_wait} seconds")
 
@@ -499,23 +530,22 @@ module Fluent::Plugin
         end
       end
 
-      rotated_tw = @tails[path]
+      new_target_info = TargetInfo.new(path, new_inode)
 
       if @follow_inodes
-        new_position_entry = @pf[target_info]
-
+        new_position_entry = @pf[new_target_info]
+        # If `refresh_watcher` find the new file before, this will not be zero.
+        # In this case, only we have to do is detaching the current tail_watcher.
         if new_position_entry.read_inode == 0
-          # When follow_inodes is true, it's not cleaned up by refresh_watcher.
-          # So it should be unwatched here explicitly.
-          rotated_tw.unwatched = true if rotated_tw
-          @tails[path] = setup_watcher(target_info, new_position_entry)
+          @tails[path] = setup_watcher(new_target_info, new_position_entry)
           @tails[path].on_notify
         end
       else
-        @tails[path] = setup_watcher(target_info, pe)
+        @tails[path] = setup_watcher(new_target_info, pe)
         @tails[path].on_notify
       end
-      detach_watcher_after_rotate_wait(rotated_tw, pe.read_inode) if rotated_tw
+
+      detach_watcher_after_rotate_wait(tail_watcher, pe.read_inode)
     end
 
     # TailWatcher#close is called by another thread at shutdown phase.
@@ -523,6 +553,10 @@ module Fluent::Plugin
     # so adding close_io argument to avoid this problem.
     # At shutdown, IOHandler's io will be released automatically after detached the event loop
     def detach_watcher(tw, ino, close_io = true)
+      if @follow_inodes && tw.ino != ino
+        log.warn("detach_watcher could be detaching an unexpected tail_watcher with a different ino.",
+                  path: tw.path, actual_ino_in_tw: tw.ino, expect_ino_to_close: ino)
+      end
       tw.watchers.each do |watcher|
         event_loop_detach(watcher)
       end
@@ -778,7 +812,7 @@ module Fluent::Plugin
       attr_accessor :group_watcher
 
       def tag
-        @parsed_tag ||= @path.tr('/', '.').gsub(/\.+/, '.').gsub(/^\./, '')
+        @parsed_tag ||= @path.tr('/', '.').squeeze('.').gsub(/^\./, '')
       end
 
       def register_watcher(watcher)
@@ -874,21 +908,21 @@ module Fluent::Plugin
 
           if watcher_needs_update
             if @follow_inodes
-              # No need to update a watcher if stat is nil (file not present), because moving to inodes will create
-              # new watcher, and old watcher will be closed by stop_watcher in refresh_watchers method
-              # don't want to swap state because we need latest read offset in pos file even after rotate_wait
-              if stat
-                target_info = TargetInfo.new(@path, stat.ino)
-                @update_watcher.call(target_info, @pe)
-              end
+              # If stat is nil (file not present), NEED to stop and discard this watcher.
+              #   When the file is disappeared but is resurrected soon, then `#refresh_watcher`
+              #   can't recognize this TailWatcher needs to be stopped.
+              #   This can happens when the file is rotated.
+              #   If a notify comes before the new file for the path is created during rotation,
+              #   then it appears as if the file was resurrected once it disappeared.
+              # Don't want to swap state because we need latest read offset in pos file even after rotate_wait
+              @update_watcher.call(self, @pe, stat&.ino)
             else
               # Permit to handle if stat is nil (file not present).
               # If a file is mv-ed and a new file is created during
               # calling `#refresh_watchers`s, and `#refresh_watchers` won't run `#start_watchers`
               # and `#stop_watchers()` for the path because `target_paths_hash`
               # always contains the path.
-              target_info = TargetInfo.new(@path, stat ? stat.ino : nil)
-              @update_watcher.call(target_info, swap_state(@pe))
+              @update_watcher.call(self, swap_state(@pe), stat&.ino)
             end
           else
             @log.info "detected rotation of #{@path}"

data/lib/fluent/plugin/in_tcp.rb

@@ -36,6 +36,10 @@ module Fluent::Plugin
     desc "The field name of the client's address."
     config_param :source_address_key, :string, default: nil
 
+    # Setting default to nil for backward compatibility
+    desc "The max bytes of message."
+    config_param :message_length_limit, :size, default: nil
+
     config_param :blocking_timeout, :time, default: 0.5
 
     desc 'The payload is read up to this character.'
@@ -102,6 +106,7 @@ module Fluent::Plugin
 
       log.info "listening tcp socket", bind: @bind, port: @port
       del_size = @delimiter.length
+      discard_till_next_delimiter = false
       if @_extract_enabled && @_extract_tag_key
         server_create(:in_tcp_server_single_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |data, conn|
           unless check_client(conn)
@@ -116,6 +121,16 @@ module Fluent::Plugin
             msg = buf[pos...i]
             pos = i + del_size
 
+            if discard_till_next_delimiter
+              discard_till_next_delimiter = false
+              next
+            end
+
+            if !@message_length_limit.nil? && @message_length_limit < msg.bytesize
+              log.info "The received data is larger than 'message_length_limit', dropped:", limit: @message_length_limit, size: msg.bytesize, head: msg[...32]
+              next
+            end
+
             @parser.parse(msg) do |time, record|
               unless time && record
                 log.warn "pattern not matched", message: msg
@@ -131,6 +146,15 @@ module Fluent::Plugin
             end
           end
           buf.slice!(0, pos) if pos > 0
+          # If the buffer size exceeds the limit here, it means that the next message will definitely exceed the limit.
+          # So we should clear the buffer here. Otherwise, it will keep storing useless data until the next delimiter comes.
+          if !@message_length_limit.nil? && @message_length_limit < buf.bytesize
+            log.info "The buffer size exceeds 'message_length_limit', cleared:", limit: @message_length_limit, size: buf.bytesize, head: buf[...32]
+            buf.clear
+            # We should discard the subsequent data until the next delimiter comes.
+            discard_till_next_delimiter = true
+            next
+          end
         end
       else
         server_create(:in_tcp_server_batch_emit, @port, bind: @bind, resolve_name: !!@source_hostname_key, send_keepalive_packet: @send_keepalive_packet) do |data, conn|
@@ -147,6 +171,16 @@ module Fluent::Plugin
             msg = buf[pos...i]
             pos = i + del_size
 
+            if discard_till_next_delimiter
+              discard_till_next_delimiter = false
+              next
+            end
+
+            if !@message_length_limit.nil? && @message_length_limit < msg.bytesize
+              log.info "The received data is larger than 'message_length_limit', dropped:", limit: @message_length_limit, size: msg.bytesize, head: msg[...32]
+              next
+            end
+
             @parser.parse(msg) do |time, record|
               unless time && record
                 log.warn "pattern not matched", message: msg
@@ -161,6 +195,15 @@ module Fluent::Plugin
           end
           router.emit_stream(@tag, es)
           buf.slice!(0, pos) if pos > 0
+          # If the buffer size exceeds the limit here, it means that the next message will definitely exceed the limit.
+          # So we should clear the buffer here. Otherwise, it will keep storing useless data until the next delimiter comes.
+          if !@message_length_limit.nil? && @message_length_limit < buf.bytesize
+            log.info "The buffer size exceeds 'message_length_limit', cleared:", limit: @message_length_limit, size: buf.bytesize, head: buf[...32]
+            buf.clear
+            # We should discard the subsequent data until the next delimiter comes.
+            discard_till_next_delimiter = true
+            next
+          end
         end
       end
     end

data/lib/fluent/plugin/out_exec_filter.rb

@@ -163,7 +163,7 @@ module Fluent::Plugin
                     0
                   elsif (@child_respawn == 'inf') || (@child_respawn == '-1')
                     -1
-                  elsif @child_respawn =~ /^\d+$/
+                  elsif /^\d+$/.match?(@child_respawn)
                     @child_respawn.to_i
                   else
                     raise ConfigError, "child_respawn option argument invalid: none(or 0), inf(or -1) or positive number"
@@ -187,7 +187,7 @@ module Fluent::Plugin
       @rr = 0
 
       exit_callback = ->(status){
-        c = @children.select{|child| child.pid == status.pid }.first
+        c = @children.find{|child| child.pid == status.pid }
         if c
           unless self.stopped?
             log.warn "child process exits with error code", code: status.to_i, status: status.exitstatus, signal: status.termsig

data/lib/fluent/plugin/output.rb

@@ -426,7 +426,7 @@ module Fluent
           end
           @secondary.acts_as_secondary(self)
           @secondary.configure(secondary_conf)
-          if (@secondary.class != SecondaryFileOutput) &&
+          if (@secondary.class.to_s != "Fluent::Plugin::SecondaryFileOutput") &&
              (self.class != @secondary.class) &&
              (@custom_format || @secondary.implement?(:custom_format))
             log.warn "Use different plugin for secondary. Check the plugin works with primary like secondary_file", primary: self.class.to_s, secondary: @secondary.class.to_s
@@ -824,7 +824,7 @@ module Fluent
             if str.include?('${tag}')
               rvalue = rvalue.gsub('${tag}', metadata.tag)
             end
-            if str =~ CHUNK_TAG_PLACEHOLDER_PATTERN
+            if CHUNK_TAG_PLACEHOLDER_PATTERN.match?(str)
               hash = {}
               tag_parts = metadata.tag.split('.')
               tag_parts.each_with_index do |part, i|

data/lib/fluent/plugin/parser_json.rb

@@ -60,7 +60,7 @@ module Fluent
       rescue LoadError => ex
         name = :yajl
         if log
-          if /\boj\z/ =~ ex.message
+          if /\boj\z/.match?(ex.message)
             log.info "Oj is not installed, and failing back to Yajl for json parser"
           else
             log.warn ex.message

data/lib/fluent/plugin_helper/event_loop.rb

@@ -99,7 +99,7 @@ module Fluent
 
       def shutdown
         @_event_loop_mutex.synchronize do
-          @_event_loop_attached_watchers.reverse.each do |w|
+          @_event_loop_attached_watchers.reverse_each do |w|
             if w.attached?
               begin
                 w.detach
@@ -116,7 +116,7 @@ module Fluent
       def after_shutdown
         timeout_at = Fluent::Clock.now + EVENT_LOOP_SHUTDOWN_TIMEOUT
         @_event_loop_mutex.synchronize do
-          @_event_loop.watchers.reverse.each do |w|
+          @_event_loop.watchers.reverse_each do |w|
             begin
               w.detach
             rescue => e

data/lib/fluent/plugin_helper/record_accessor.rb

@@ -119,7 +119,7 @@ module Fluent
         def self.validate_dot_keys(keys)
           keys.each { |key|
             next unless key.is_a?(String)
-            if /\s+/.match(key)
+            if /\s+/.match?(key)
               raise Fluent::ConfigError, "whitespace character is not allowed in dot notation. Use bracket notation: #{key}"
             end
           }

data/lib/fluent/plugin_helper/server.rb

@@ -545,6 +545,10 @@ module Fluent
               data = @sock.recv(@max_bytes, @flags)
             rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET, IOError, Errno::EBADF
               return
+            rescue Errno::EMSGSIZE
+              # Windows ONLY: This happens when the data size is larger than `@max_bytes`.
+              @log.info "A received data was ignored since it was too large."
+              return
             end
             @callback.call(data)
           rescue => e
@@ -558,6 +562,10 @@ module Fluent
               data, addr = @sock.recvfrom(@max_bytes)
             rescue Errno::EAGAIN, Errno::EWOULDBLOCK, Errno::EINTR, Errno::ECONNRESET, IOError, Errno::EBADF
               return
+            rescue Errno::EMSGSIZE
+              # Windows ONLY: This happens when the data size is larger than `@max_bytes`.
+              @log.info "A received data was ignored since it was too large."
+              return
             end
             @callback.call(data, UDPCallbackSocket.new(@sock, addr, close_socket: @close_socket))
           rescue => e

data/lib/fluent/plugin_helper/thread.rb

@@ -101,16 +101,16 @@ module Fluent
       end
 
       def thread_exist?(title)
-        @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.size > 0
+        @_threads.values.count{|thread| title == thread[:_fluentd_plugin_helper_thread_title] } > 0
       end
 
       def thread_started?(title)
-        t = @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.first
+        t = @_threads.values.find{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }
         t && t[:_fluentd_plugin_helper_thread_started]
       end
 
       def thread_running?(title)
-        t = @_threads.values.select{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }.first
+        t = @_threads.values.find{|thread| title == thread[:_fluentd_plugin_helper_thread_title] }
         t && t[:_fluentd_plugin_helper_thread_running]
       end
 

data/lib/fluent/plugin_id.rb

@@ -49,7 +49,7 @@ module Fluent
         # Thread::Backtrace::Location#path returns base filename or absolute path.
         # #absolute_path returns absolute_path always.
         # https://bugs.ruby-lang.org/issues/12159
-        if location.absolute_path =~ /\/test_[^\/]+\.rb$/ # location.path =~ /test_.+\.rb$/
+        if /\/test_[^\/]+\.rb$/.match?(location.absolute_path) # location.path =~ /test_.+\.rb$/
           return true
         end
       end

data/lib/fluent/supervisor.rb

@@ -697,7 +697,7 @@ module Fluent
       actual_log_path = @log_path
 
       # We need to prepare a unique path for each worker since Windows locks files.
-      if Fluent.windows? && rotate
+      if Fluent.windows? && rotate && @log_path && @log_path != "-"
         actual_log_path = Fluent::Log.per_process_path(@log_path, process_type, worker_id)
       end
 

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.16.0'
+  VERSION = '1.16.2'
 
 end

data/templates/new_gem/test/helper.rb.erb

@@ -1,4 +1,3 @@
-$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
 require "test-unit"
 require "fluent/test"
 require "fluent/test/driver/<%= type %>"

data/test/plugin/in_tail/test_position_file.rb

@@ -26,6 +26,10 @@ class IntailPositionFileTest < Test::Unit::TestCase
     "valid_path" => Fluent::Plugin::TailInput::TargetInfo.new("valid_path", 1),
     "inode23bit" => Fluent::Plugin::TailInput::TargetInfo.new("inode23bit", 0),
   }
+  TEST_CONTENT_INODES = {
+    1 => Fluent::Plugin::TailInput::TargetInfo.new("valid_path", 1),
+    0 => Fluent::Plugin::TailInput::TargetInfo.new("inode23bit", 0),
+  }
 
   def write_data(f, content)
     f.write(content)
@@ -221,7 +225,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
   end
 
   sub_test_case '#unwatch' do
-    test 'deletes entry by path' do
+    test 'unwatch entry by path' do
       write_data(@file, TEST_CONTENT)
       pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, logger: $log)
       inode1 = File.stat(@file).ino
@@ -239,6 +243,32 @@ class IntailPositionFileTest < Test::Unit::TestCase
 
       assert_not_equal p1, p2
     end
+
+    test 'unwatch entries by inode' do
+      write_data(@file, TEST_CONTENT)
+      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, true, TEST_CONTENT_INODES, logger: $log)
+
+      existing_targets = TEST_CONTENT_INODES.select do |inode, target_info|
+        inode == 1
+      end
+      pe_to_unwatch = pf[TEST_CONTENT_INODES[0]]
+
+      pf.unwatch_removed_targets(existing_targets)
+
+      assert_equal(
+        {
+          map_keys: [TEST_CONTENT_INODES[1].ino],
+          unwatched_pe_pos: Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION,
+        },
+        {
+          map_keys: pf.instance_variable_get(:@map).keys,
+          unwatched_pe_pos: pe_to_unwatch.read_pos,
+        }
+      )
+
+      unwatched_pe_retaken = pf[TEST_CONTENT_INODES[0]]
+      assert_not_equal pe_to_unwatch, unwatched_pe_retaken
+    end
   end
 
   sub_test_case 'FilePositionEntry' do

data/test/plugin/test_base.rb

@@ -108,7 +108,7 @@ class BaseTest < Test::Unit::TestCase
     @p.extend m
     assert_equal [], logger.logs
 
-    ret = @p.string_safe_encoding("abc\xff.\x01f"){|s| s.split(/\./) }
+    ret = @p.string_safe_encoding("abc\xff.\x01f"){|s| s.split(".") }
     assert_equal ['abc?', "\u0001f"], ret
     assert_equal 1, logger.logs.size
     assert{ logger.logs.first.include?("invalid byte sequence is replaced in ") }

data/test/plugin/test_buffer_chunk.rb

@@ -57,6 +57,17 @@ class BufferChunkTest < Test::Unit::TestCase
       assert chunk.respond_to?(:msgpack_each)
     end
 
+    test 'unpacker arg is not implemented for ChunkMessagePackEventStreamer' do
+      meta = Object.new
+      chunk = Fluent::Plugin::Buffer::Chunk.new(meta)
+      chunk.extend Fluent::ChunkMessagePackEventStreamer
+
+      unpacker = Fluent::MessagePackFactory.thread_local_msgpack_unpacker
+
+      assert_raise(NotImplementedError){ chunk.each(unpacker: unpacker) }
+      assert_raise(NotImplementedError){ chunk.msgpack_each(unpacker: unpacker) }
+    end
+
     test 'some methods raise ArgumentError with an option of `compressed: :gzip` and without extending Compressble`' do
       meta = Object.new
       chunk = Fluent::Plugin::Buffer::Chunk.new(meta)

data/test/plugin/test_in_forward.rb

@@ -367,7 +367,7 @@ class ForwardInputTest < Test::Unit::TestCase
       end
 
       logs = d.instance.log.out.logs
-      assert{ logs.select{|line| line =~ /skip invalid event/ }.size == 2 }
+      assert{ logs.count{|line| line =~ /skip invalid event/ } == 2 }
 
       d.instance_shutdown
     end
@@ -593,10 +593,10 @@ class ForwardInputTest < Test::Unit::TestCase
 
       # check log
       logs = d.instance.log.logs
-      assert_equal 1, logs.select{|line|
+      assert_equal 1, logs.count{|line|
         line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_warn_limit':/ &&
         line =~ / tag="test.tag" host="#{LOCALHOST_HOSTNAME}" limit=16777216 size=16777501/
-      }.size, "large chunk warning is not logged"
+      }, "large chunk warning is not logged"
 
       d.instance_shutdown
     end
@@ -619,10 +619,10 @@ class ForwardInputTest < Test::Unit::TestCase
 
       # check log
       logs = d.instance.log.logs
-      assert_equal 1, logs.select{ |line|
+      assert_equal 1, logs.count{ |line|
         line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_warn_limit':/ &&
         line =~ / tag="test.tag" host="#{LOCALHOST_HOSTNAME}" limit=16777216 size=16777501/
-      }.size, "large chunk warning is not logged"
+      }, "large chunk warning is not logged"
 
       d.instance_shutdown
     end
@@ -653,10 +653,10 @@ class ForwardInputTest < Test::Unit::TestCase
 
       # check log
       logs = d.instance.log.logs
-      assert_equal 1, logs.select{|line|
+      assert_equal 1, logs.count{|line|
         line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_limit', dropped:/ &&
         line =~ / tag="test.tag" host="#{LOCALHOST_HOSTNAME}" limit=33554432 size=33554989/
-      }.size, "large chunk warning is not logged"
+      }, "large chunk warning is not logged"
 
       d.instance_shutdown
     end
@@ -676,9 +676,9 @@ class ForwardInputTest < Test::Unit::TestCase
 
       # check log
       logs = d.instance.log.logs
-      assert_equal 1, logs.select{|line|
+      assert_equal 1, logs.count{|line|
         line =~ / \[warn\]: incoming chunk is broken: host="#{LOCALHOST_HOSTNAME}" msg=#{data.inspect}/
-      }.size, "should not accept broken chunk"
+      }, "should not accept broken chunk"
 
       d.instance_shutdown
     end