fluentd 1.16.0 → 1.16.7

data/lib/fluent/plugin/buffer.rb

@@ -417,7 +417,7 @@ module Fluent
             if c.staged? && (enqueue || chunk_size_full?(c))
               m = c.metadata
               enqueue_chunk(m)
-              if unstaged_chunks[m]
+              if unstaged_chunks[m] && !unstaged_chunks[m].empty?
                 u = unstaged_chunks[m].pop
                 u.synchronize do
                   if u.unstaged? && !chunk_size_full?(u)
@@ -580,7 +580,7 @@ module Fluent
           chunk = @dequeued.delete(chunk_id)
           return false unless chunk # already purged by other thread
           @queue.unshift(chunk)
-          log.trace "chunk taken back", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: chunk.metadata
+          log.on_trace { log.trace "chunk taken back", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: chunk.metadata }
           @queued_num[chunk.metadata] += 1 # BUG if nil
           @dequeued_num[chunk.metadata] -= 1
         end
@@ -610,7 +610,7 @@ module Fluent
             @queued_num.delete(metadata)
             @dequeued_num.delete(metadata)
           end
-          log.trace "chunk purged", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata
+          log.on_trace { log.trace "chunk purged", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata }
         end
 
         nil
@@ -728,7 +728,6 @@ module Fluent
 
       def write_step_by_step(metadata, data, format, splits_count, &block)
         splits = []
-        errors = []
         if splits_count > data.size
           splits_count = data.size
         end
@@ -749,16 +748,14 @@ module Fluent
         modified_chunks = []
         modified_metadata = metadata
         get_next_chunk = ->(){
-          c = if staged_chunk_used
-                # Staging new chunk here is bad idea:
-                # Recovering whole state including newly staged chunks is much harder than current implementation.
-                modified_metadata = modified_metadata.dup_next
-                generate_chunk(modified_metadata)
-              else
-                synchronize { @stage[modified_metadata] ||= generate_chunk(modified_metadata).staged! }
-              end
-          modified_chunks << c
-          c
+          if staged_chunk_used
+            # Staging new chunk here is bad idea:
+            # Recovering whole state including newly staged chunks is much harder than current implementation.
+            modified_metadata = modified_metadata.dup_next
+            generate_chunk(modified_metadata)
+          else
+            synchronize { @stage[modified_metadata] ||= generate_chunk(modified_metadata).staged! }
+          end
         }
 
         writing_splits_index = 0
@@ -766,104 +763,116 @@ module Fluent
 
         while writing_splits_index < splits.size
           chunk = get_next_chunk.call
-          chunk.synchronize do
-            raise ShouldRetry unless chunk.writable?
-            staged_chunk_used = true if chunk.staged?
-
-            original_bytesize = committed_bytesize = chunk.bytesize
-            begin
-              while writing_splits_index < splits.size
-                split = splits[writing_splits_index]
-                formatted_split = format ? format.call(split) : nil
+          errors = []
+          # The chunk must be locked until being passed to &block.
+          chunk.mon_enter
+          modified_chunks << {chunk: chunk, adding_bytesize: 0, errors: errors}
 
-                if split.size == 1 # Check BufferChunkOverflowError
-                  determined_bytesize = nil
-                  if @compress != :text
-                    determined_bytesize = nil
-                  elsif formatted_split
-                    determined_bytesize = formatted_split.bytesize
-                  elsif split.first.respond_to?(:bytesize)
-                    determined_bytesize = split.first.bytesize
-                  end
+          raise ShouldRetry unless chunk.writable?
+          staged_chunk_used = true if chunk.staged?
 
-                  if determined_bytesize && determined_bytesize > @chunk_limit_size
-                    # It is a obvious case that BufferChunkOverflowError should be raised here.
-                    # But if it raises here, already processed 'split' or
-                    # the proceeding 'split' will be lost completely.
-                    # So it is a last resort to delay raising such a exception
-                    errors << "a #{determined_bytesize} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
-                    writing_splits_index += 1
-                    next
-                  end
+          original_bytesize = committed_bytesize = chunk.bytesize
+          begin
+            while writing_splits_index < splits.size
+              split = splits[writing_splits_index]
+              formatted_split = format ? format.call(split) : nil
 
-                  if determined_bytesize.nil? || chunk.bytesize + determined_bytesize > @chunk_limit_size
-                    # The split will (might) cause size over so keep already processed
-                    # 'split' content here (allow performance regression a bit).
-                    chunk.commit
-                    committed_bytesize = chunk.bytesize
-                  end
+              if split.size == 1 # Check BufferChunkOverflowError
+                determined_bytesize = nil
+                if @compress != :text
+                  determined_bytesize = nil
+                elsif formatted_split
+                  determined_bytesize = formatted_split.bytesize
+                elsif split.first.respond_to?(:bytesize)
+                  determined_bytesize = split.first.bytesize
                 end
 
-                if format
-                  chunk.concat(formatted_split, split.size)
-                else
-                  chunk.append(split, compress: @compress)
+                if determined_bytesize && determined_bytesize > @chunk_limit_size
+                  # It is a obvious case that BufferChunkOverflowError should be raised here.
+                  # But if it raises here, already processed 'split' or
+                  # the proceeding 'split' will be lost completely.
+                  # So it is a last resort to delay raising such a exception
+                  errors << "a #{determined_bytesize} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
+                  writing_splits_index += 1
+                  next
                 end
-                adding_bytes = chunk.bytesize - committed_bytesize
 
-                if chunk_size_over?(chunk) # split size is larger than difference between size_full? and size_over?
-                  chunk.rollback
+                if determined_bytesize.nil? || chunk.bytesize + determined_bytesize > @chunk_limit_size
+                  # The split will (might) cause size over so keep already processed
+                  # 'split' content here (allow performance regression a bit).
+                  chunk.commit
                   committed_bytesize = chunk.bytesize
+                end
+              end
 
-                  if split.size == 1 # Check BufferChunkOverflowError again
-                    if adding_bytes > @chunk_limit_size
-                      errors << "concatenated/appended a #{adding_bytes} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
-                      writing_splits_index += 1
-                      next
-                    else
-                      # As already processed content is kept after rollback, then unstaged chunk should be queued.
-                      # After that, re-process current split again.
-                      # New chunk should be allocated, to do it, modify @stage and so on.
-                      synchronize { @stage.delete(modified_metadata) }
-                      staged_chunk_used = false
-                      chunk.unstaged!
-                      break
-                    end
-                  end
+              if format
+                chunk.concat(formatted_split, split.size)
+              else
+                chunk.append(split, compress: @compress)
+              end
+              adding_bytes = chunk.bytesize - committed_bytesize
 
-                  if chunk_size_full?(chunk) || split.size == 1
-                    enqueue_chunk_before_retry = true
+              if chunk_size_over?(chunk) # split size is larger than difference between size_full? and size_over?
+                chunk.rollback
+                committed_bytesize = chunk.bytesize
+
+                if split.size == 1 # Check BufferChunkOverflowError again
+                  if adding_bytes > @chunk_limit_size
+                    errors << "concatenated/appended a #{adding_bytes} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
+                    writing_splits_index += 1
+                    next
                   else
-                    splits_count *= 10
+                    # As already processed content is kept after rollback, then unstaged chunk should be queued.
+                    # After that, re-process current split again.
+                    # New chunk should be allocated, to do it, modify @stage and so on.
+                    synchronize { @stage.delete(modified_metadata) }
+                    staged_chunk_used = false
+                    chunk.unstaged!
+                    break
                   end
+                end
 
-                  raise ShouldRetry
+                if chunk_size_full?(chunk) || split.size == 1
+                  enqueue_chunk_before_retry = true
+                else
+                  splits_count *= 10
                 end
 
-                writing_splits_index += 1
+                raise ShouldRetry
+              end
+
+              writing_splits_index += 1
 
-                if chunk_size_full?(chunk)
-                  break
-                end
+              if chunk_size_full?(chunk)
+                break
               end
-            rescue
-              chunk.purge if chunk.unstaged? # unstaged chunk will leak unless purge it
-              raise
             end
-
-            block.call(chunk, chunk.bytesize - original_bytesize, errors)
-            errors = []
+          rescue
+            chunk.purge if chunk.unstaged? # unstaged chunk will leak unless purge it
+            raise
           end
+
+          modified_chunks.last[:adding_bytesize] = chunk.bytesize - original_bytesize
+        end
+        modified_chunks.each do |data|
+          block.call(data[:chunk], data[:adding_bytesize], data[:errors])
         end
       rescue ShouldRetry
-        modified_chunks.each do |mc|
-          mc.rollback rescue nil
-          if mc.unstaged?
-            mc.purge rescue nil
+        modified_chunks.each do |data|
+          chunk = data[:chunk]
+          chunk.rollback rescue nil
+          if chunk.unstaged?
+            chunk.purge rescue nil
           end
+          chunk.mon_exit rescue nil
         end
         enqueue_chunk(metadata) if enqueue_chunk_before_retry
         retry
+      ensure
+        modified_chunks.each do |data|
+          chunk = data[:chunk]
+          chunk.mon_exit
+        end
       end
 
       STATS_KEYS = [

data/lib/fluent/plugin/filter_record_transformer.rb

@@ -316,7 +316,7 @@ module Fluent::Plugin
         end
 
         (Object.instance_methods).each do |m|
-          undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member|^class$/
+          undef_method m unless /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member|^class$/.match?(m.to_s)
         end
       end
     end

data/lib/fluent/plugin/in_forward.rb

@@ -430,7 +430,7 @@ module Fluent::Plugin
       end
       _ping, hostname, shared_key_salt, shared_key_hexdigest, username, password_digest = message
 
-      node = @nodes.select{|n| n[:address].include?(remote_addr) rescue false }.first
+      node = @nodes.find{|n| n[:address].include?(remote_addr) rescue false }
       if !node && !@security.allow_anonymous_source
         log.warn "Anonymous client disallowed", address: remote_addr, hostname: hostname
         return false, "anonymous source host '#{remote_addr}' denied", nil

data/lib/fluent/plugin/in_http.rb

@@ -428,7 +428,7 @@ module Fluent::Plugin
         @content_type = ""
         @content_encoding = ""
         headers.each_pair {|k,v|
-          @env["HTTP_#{k.gsub('-','_').upcase}"] = v
+          @env["HTTP_#{k.tr('-','_').upcase}"] = v
           case k
           when /\AExpect\z/i
             expect = v
@@ -439,9 +439,9 @@ module Fluent::Plugin
           when /\AContent-Encoding\Z/i
             @content_encoding = v
           when /\AConnection\Z/i
-            if v =~ /close/i
+            if /close/i.match?(v)
               @keep_alive = false
-            elsif v =~ /Keep-alive/i
+            elsif /Keep-alive/i.match?(v)
               @keep_alive = true
             end
           when /\AOrigin\Z/i
@@ -566,16 +566,16 @@ module Fluent::Plugin
 
         if @format_name != 'default'
           params[EVENT_RECORD_PARAMETER] = @body
-        elsif @content_type =~ /^application\/x-www-form-urlencoded/
+        elsif /^application\/x-www-form-urlencoded/.match?(@content_type)
           params.update WEBrick::HTTPUtils.parse_query(@body)
         elsif @content_type =~ /^multipart\/form-data; boundary=(.+)/
           boundary = WEBrick::HTTPUtils.dequote($1)
           params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)
-        elsif @content_type =~ /^application\/json/
+        elsif /^application\/json/.match?(@content_type)
           params['json'] = @body
-        elsif @content_type =~ /^application\/msgpack/
+        elsif /^application\/msgpack/.match?(@content_type)
           params['msgpack'] = @body
-        elsif @content_type =~ /^application\/x-ndjson/
+        elsif /^application\/x-ndjson/.match?(@content_type)
           params['ndjson'] = @body
         end
         path_info = uri.path
@@ -585,7 +585,7 @@ module Fluent::Plugin
           query_params = WEBrick::HTTPUtils.parse_query(uri.query)
 
           query_params.each_pair {|k,v|
-            params["QUERY_#{k.gsub('-','_').upcase}"] = v
+            params["QUERY_#{k.tr('-','_').upcase}"] = v
           }
         end
 

data/lib/fluent/plugin/in_sample.rb

@@ -64,7 +64,7 @@ module Fluent::Plugin
     def configure(conf)
       super
       @sample_index = 0
-      config = conf.elements.select{|e| e.name == 'storage' }.first
+      config = conf.elements.find{|e| e.name == 'storage' }
       @storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)
     end
 

data/lib/fluent/plugin/in_tail/position_file.rb

@@ -53,10 +53,16 @@ module Fluent::Plugin
         }
       end
 
+      def unwatch_removed_targets(existing_targets)
+        @map.reject { |key, entry|
+          existing_targets.key?(key)
+        }.each_key { |key|
+          unwatch_key(key)
+        }
+      end
+
       def unwatch(target_info)
-        if (entry = @map.delete(@follow_inodes ? target_info.ino : target_info.path))
-          entry.update_pos(UNWATCHED_POSITION)
-        end
+        unwatch_key(@follow_inodes ? target_info.ino : target_info.path)
       end
 
       def load(existing_targets = nil)
@@ -96,6 +102,7 @@ module Fluent::Plugin
         end
 
         entries = fetch_compacted_entries
+        @logger&.debug "Compacted entries: ", entries.keys
 
         @file_mutex.synchronize do
           if last_modified == @file.mtime && size == @file.size
@@ -117,17 +124,31 @@ module Fluent::Plugin
 
       private
 
+      def unwatch_key(key)
+        if (entry = @map.delete(key))
+          entry.update_pos(UNWATCHED_POSITION)
+        end
+      end
+
       def compact(existing_targets = nil)
         @file_mutex.synchronize do
-          entries = fetch_compacted_entries(existing_targets).values.map(&:to_entry_fmt)
+          entries = fetch_compacted_entries
+          @logger&.debug "Compacted entries: ", entries.keys
+
+          if existing_targets
+            entries = remove_deleted_files_entries(entries, existing_targets)
+            @logger&.debug "Remove missing entries.",
+                           existing_targets: existing_targets.keys,
+                           entries_after_removing: entries.keys
+          end
 
           @file.pos = 0
           @file.truncate(0)
-          @file.write(entries.join)
+          @file.write(entries.values.map(&:to_entry_fmt).join)
         end
       end
 
-      def fetch_compacted_entries(existing_targets = nil)
+      def fetch_compacted_entries
         entries = {}
 
         @file.pos = 0
@@ -145,31 +166,24 @@ module Fluent::Plugin
           if pos == UNWATCHED_POSITION
             @logger.debug "Remove unwatched line from pos_file: #{line}" if @logger
           else
-            if entries.include?(path)
-              @logger.warn("#{path} already exists. use latest one: deleted #{entries[path]}") if @logger
-            end
-
             if @follow_inodes
+              @logger&.warn("#{path} (inode: #{ino}) already exists. use latest one: deleted #{entries[ino]}") if entries.include?(ino)
               entries[ino] = Entry.new(path, pos, ino, file_pos + path.bytesize + 1)
             else
+              @logger&.warn("#{path} already exists. use latest one: deleted #{entries[path]}") if entries.include?(path)
               entries[path] = Entry.new(path, pos, ino, file_pos + path.bytesize + 1)
             end
             file_pos += line.size
           end
         end
 
-        entries = remove_deleted_files_entries(entries, existing_targets)
         entries
       end
 
       def remove_deleted_files_entries(existent_entries, existing_targets)
-        if existing_targets
-          existent_entries.select { |path_or_ino|
-            existing_targets.key?(path_or_ino)
-          }
-        else
-          existent_entries
-        end
+        existent_entries.select { |path_or_ino|
+          existing_targets.key?(path_or_ino)
+        }
       end
     end