fluentd 1.16.3 → 1.17.0

data/lib/fluent/plugin/buffer.rb

@@ -580,7 +580,7 @@ module Fluent
           chunk = @dequeued.delete(chunk_id)
           return false unless chunk # already purged by other thread
           @queue.unshift(chunk)
-          log.trace "chunk taken back", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: chunk.metadata
+          log.on_trace { log.trace "chunk taken back", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: chunk.metadata }
           @queued_num[chunk.metadata] += 1 # BUG if nil
           @dequeued_num[chunk.metadata] -= 1
         end
@@ -610,7 +610,7 @@ module Fluent
             @queued_num.delete(metadata)
             @dequeued_num.delete(metadata)
           end
-          log.trace "chunk purged", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata
+          log.on_trace { log.trace "chunk purged", instance: self.object_id, chunk_id: dump_unique_id_hex(chunk_id), metadata: metadata }
         end
 
         nil
@@ -728,7 +728,6 @@ module Fluent
 
       def write_step_by_step(metadata, data, format, splits_count, &block)
         splits = []
-        errors = []
         if splits_count > data.size
           splits_count = data.size
         end
@@ -749,16 +748,14 @@ module Fluent
         modified_chunks = []
         modified_metadata = metadata
         get_next_chunk = ->(){
-          c = if staged_chunk_used
-                # Staging new chunk here is bad idea:
-                # Recovering whole state including newly staged chunks is much harder than current implementation.
-                modified_metadata = modified_metadata.dup_next
-                generate_chunk(modified_metadata)
-              else
-                synchronize { @stage[modified_metadata] ||= generate_chunk(modified_metadata).staged! }
-              end
-          modified_chunks << c
-          c
+          if staged_chunk_used
+            # Staging new chunk here is bad idea:
+            # Recovering whole state including newly staged chunks is much harder than current implementation.
+            modified_metadata = modified_metadata.dup_next
+            generate_chunk(modified_metadata)
+          else
+            synchronize { @stage[modified_metadata] ||= generate_chunk(modified_metadata).staged! }
+          end
         }
 
         writing_splits_index = 0
@@ -766,104 +763,116 @@ module Fluent
 
         while writing_splits_index < splits.size
           chunk = get_next_chunk.call
-          chunk.synchronize do
-            raise ShouldRetry unless chunk.writable?
-            staged_chunk_used = true if chunk.staged?
-
-            original_bytesize = committed_bytesize = chunk.bytesize
-            begin
-              while writing_splits_index < splits.size
-                split = splits[writing_splits_index]
-                formatted_split = format ? format.call(split) : nil
+          errors = []
+          # The chunk must be locked until being passed to &block.
+          chunk.mon_enter
+          modified_chunks << {chunk: chunk, adding_bytesize: 0, errors: errors}
 
-                if split.size == 1 # Check BufferChunkOverflowError
-                  determined_bytesize = nil
-                  if @compress != :text
-                    determined_bytesize = nil
-                  elsif formatted_split
-                    determined_bytesize = formatted_split.bytesize
-                  elsif split.first.respond_to?(:bytesize)
-                    determined_bytesize = split.first.bytesize
-                  end
+          raise ShouldRetry unless chunk.writable?
+          staged_chunk_used = true if chunk.staged?
 
-                  if determined_bytesize && determined_bytesize > @chunk_limit_size
-                    # It is a obvious case that BufferChunkOverflowError should be raised here.
-                    # But if it raises here, already processed 'split' or
-                    # the proceeding 'split' will be lost completely.
-                    # So it is a last resort to delay raising such a exception
-                    errors << "a #{determined_bytesize} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
-                    writing_splits_index += 1
-                    next
-                  end
+          original_bytesize = committed_bytesize = chunk.bytesize
+          begin
+            while writing_splits_index < splits.size
+              split = splits[writing_splits_index]
+              formatted_split = format ? format.call(split) : nil
 
-                  if determined_bytesize.nil? || chunk.bytesize + determined_bytesize > @chunk_limit_size
-                    # The split will (might) cause size over so keep already processed
-                    # 'split' content here (allow performance regression a bit).
-                    chunk.commit
-                    committed_bytesize = chunk.bytesize
-                  end
+              if split.size == 1 # Check BufferChunkOverflowError
+                determined_bytesize = nil
+                if @compress != :text
+                  determined_bytesize = nil
+                elsif formatted_split
+                  determined_bytesize = formatted_split.bytesize
+                elsif split.first.respond_to?(:bytesize)
+                  determined_bytesize = split.first.bytesize
                 end
 
-                if format
-                  chunk.concat(formatted_split, split.size)
-                else
-                  chunk.append(split, compress: @compress)
+                if determined_bytesize && determined_bytesize > @chunk_limit_size
+                  # It is a obvious case that BufferChunkOverflowError should be raised here.
+                  # But if it raises here, already processed 'split' or
+                  # the proceeding 'split' will be lost completely.
+                  # So it is a last resort to delay raising such a exception
+                  errors << "a #{determined_bytesize} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
+                  writing_splits_index += 1
+                  next
                 end
-                adding_bytes = chunk.bytesize - committed_bytesize
 
-                if chunk_size_over?(chunk) # split size is larger than difference between size_full? and size_over?
-                  chunk.rollback
+                if determined_bytesize.nil? || chunk.bytesize + determined_bytesize > @chunk_limit_size
+                  # The split will (might) cause size over so keep already processed
+                  # 'split' content here (allow performance regression a bit).
+                  chunk.commit
                   committed_bytesize = chunk.bytesize
+                end
+              end
 
-                  if split.size == 1 # Check BufferChunkOverflowError again
-                    if adding_bytes > @chunk_limit_size
-                      errors << "concatenated/appended a #{adding_bytes} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
-                      writing_splits_index += 1
-                      next
-                    else
-                      # As already processed content is kept after rollback, then unstaged chunk should be queued.
-                      # After that, re-process current split again.
-                      # New chunk should be allocated, to do it, modify @stage and so on.
-                      synchronize { @stage.delete(modified_metadata) }
-                      staged_chunk_used = false
-                      chunk.unstaged!
-                      break
-                    end
-                  end
+              if format
+                chunk.concat(formatted_split, split.size)
+              else
+                chunk.append(split, compress: @compress)
+              end
+              adding_bytes = chunk.bytesize - committed_bytesize
 
-                  if chunk_size_full?(chunk) || split.size == 1
-                    enqueue_chunk_before_retry = true
+              if chunk_size_over?(chunk) # split size is larger than difference between size_full? and size_over?
+                chunk.rollback
+                committed_bytesize = chunk.bytesize
+
+                if split.size == 1 # Check BufferChunkOverflowError again
+                  if adding_bytes > @chunk_limit_size
+                    errors << "concatenated/appended a #{adding_bytes} bytes record (nth: #{writing_splits_index}) is larger than buffer chunk limit size (#{@chunk_limit_size})"
+                    writing_splits_index += 1
+                    next
                   else
-                    splits_count *= 10
+                    # As already processed content is kept after rollback, then unstaged chunk should be queued.
+                    # After that, re-process current split again.
+                    # New chunk should be allocated, to do it, modify @stage and so on.
+                    synchronize { @stage.delete(modified_metadata) }
+                    staged_chunk_used = false
+                    chunk.unstaged!
+                    break
                   end
+                end
 
-                  raise ShouldRetry
+                if chunk_size_full?(chunk) || split.size == 1
+                  enqueue_chunk_before_retry = true
+                else
+                  splits_count *= 10
                 end
 
-                writing_splits_index += 1
+                raise ShouldRetry
+              end
+
+              writing_splits_index += 1
 
-                if chunk_size_full?(chunk)
-                  break
-                end
+              if chunk_size_full?(chunk)
+                break
               end
-            rescue
-              chunk.purge if chunk.unstaged? # unstaged chunk will leak unless purge it
-              raise
             end
-
-            block.call(chunk, chunk.bytesize - original_bytesize, errors)
-            errors = []
+          rescue
+            chunk.purge if chunk.unstaged? # unstaged chunk will leak unless purge it
+            raise
           end
+
+          modified_chunks.last[:adding_bytesize] = chunk.bytesize - original_bytesize
+        end
+        modified_chunks.each do |data|
+          block.call(data[:chunk], data[:adding_bytesize], data[:errors])
         end
       rescue ShouldRetry
-        modified_chunks.each do |mc|
-          mc.rollback rescue nil
-          if mc.unstaged?
-            mc.purge rescue nil
+        modified_chunks.each do |data|
+          chunk = data[:chunk]
+          chunk.rollback rescue nil
+          if chunk.unstaged?
+            chunk.purge rescue nil
           end
+          chunk.mon_exit rescue nil
         end
         enqueue_chunk(metadata) if enqueue_chunk_before_retry
         retry
+      ensure
+        modified_chunks.each do |data|
+          chunk = data[:chunk]
+          chunk.mon_exit
+        end
       end
 
       STATS_KEYS = [

data/lib/fluent/plugin/filter_parser.rb

@@ -70,6 +70,13 @@ module Fluent::Plugin
         end
       end
       begin
+        # Note: https://github.com/fluent/fluentd/issues/4100
+        # If the parser returns multiple records from one raw_value,
+        # this returns only the first one record.
+        # This should be fixed in the future version.
+        result_time = nil
+        result_record = nil
+
         @parser.parse(raw_value) do |t, values|
           if values
             t = if @reserve_time
@@ -79,20 +86,31 @@ module Fluent::Plugin
                 end
             @accessor.delete(record) if @remove_key_name_field
             r = handle_parsed(tag, record, t, values)
-            return t, r
+
+            if result_record.nil?
+              result_time = t
+              result_record = r
+            else
+              if @emit_invalid_record_to_error
+                router.emit_error_event(tag, t, r, Fluent::Plugin::Parser::ParserError.new(
+                  "Could not emit the event. The parser returned multiple results, but currently filter_parser plugin only returns the first parsed result. Raw data: '#{raw_value}'"
+                ))
+              end
+            end
           else
             if @emit_invalid_record_to_error
               router.emit_error_event(tag, time, record, Fluent::Plugin::Parser::ParserError.new("pattern not matched with data '#{raw_value}'"))
             end
-            if @reserve_data
-              t = time
-              r = handle_parsed(tag, record, time, {})
-              return t, r
-            else
-              return FAILED_RESULT
-            end
+
+            next unless @reserve_data
+            next unless result_record.nil?
+
+            result_time = time
+            result_record = handle_parsed(tag, record, time, {})
           end
         end
+
+        return result_time, result_record
       rescue Fluent::Plugin::Parser::ParserError => e
         if @emit_invalid_record_to_error
           raise e

data/lib/fluent/plugin/in_http.rb

@@ -203,54 +203,24 @@ module Fluent::Plugin
       begin
         path = path_info[1..-1]  # remove /
         tag = path.split('/').join('.')
-        record_time, record = parse_params(params)
 
-        # Skip nil record
-        if record.nil?
-          log.debug { "incoming event is invalid: path=#{path_info} params=#{params.to_json}" }
-          if @respond_with_empty_img
-            return RESPONSE_IMG
-          else
-            if @use_204_response
-              return RESPONSE_204
-            else
-              return RESPONSE_200
-            end
+        mes = Fluent::MultiEventStream.new
+        parse_params(params) do |record_time, record|
+          if record.nil?
+            log.debug { "incoming event is invalid: path=#{path_info} params=#{params.to_json}" }
+            next
           end
-        end
 
-        mes = nil
-        # Support batched requests
-        if record.is_a?(Array)
-          mes = Fluent::MultiEventStream.new
-          record.each do |single_record|
-            add_params_to_record(single_record, params)
-
-            if param_time = params['time']
-              param_time = param_time.to_f
-              single_time = param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
-            elsif @custom_parser
-              single_time = @custom_parser.parse_time(single_record)
-              single_time, single_record = @custom_parser.convert_values(single_time, single_record)
-            else
-              single_time = convert_time_field(single_record)
-            end
-
-            mes.add(single_time, single_record)
-          end
-        else
           add_params_to_record(record, params)
 
           time = if param_time = params['time']
                    param_time = param_time.to_f
                    param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
                  else
-                   if record_time.nil?
-                     convert_time_field(record)
-                   else
-                     record_time
-                   end
+                   record_time.nil? ? convert_time_field(record) : record_time
                  end
+
+          mes.add(time, record)
         end
       rescue => e
         if @dump_error_log
@@ -261,11 +231,7 @@ module Fluent::Plugin
 
       # TODO server error
       begin
-        if mes
-          router.emit_stream(tag, mes)
-        else
-          router.emit(tag, time, record)
-        end
+        router.emit_stream(tag, mes) unless mes.empty?
       rescue => e
         if @dump_error_log
           log.error "failed to emit data", error: e
@@ -308,20 +274,18 @@ module Fluent::Plugin
     def parse_params_default(params)
       if msgpack = params['msgpack']
         @parser_msgpack.parse(msgpack) do |_time, record|
-          return nil, record
+          yield nil, record
         end
       elsif js = params['json']
         @parser_json.parse(js) do |_time, record|
-          return nil, record
+          yield nil, record
         end
       elsif ndjson = params['ndjson']
-        events = []
         ndjson.split(/\r?\n/).each do |js|
           @parser_json.parse(js) do |_time, record|
-            events.push(record)
+            yield nil, record
           end
         end
-        return nil, events
       else
         raise "'json', 'ndjson' or 'msgpack' parameter is required"
       end
@@ -329,10 +293,9 @@ module Fluent::Plugin
 
     def parse_params_with_parser(params)
       if content = params[EVENT_RECORD_PARAMETER]
-        @custom_parser.parse(content) { |time, record|
-          raise "Received event is not #{@format_name}: #{content}" if record.nil?
-          return time, record
-        }
+        @custom_parser.parse(content) do |time, record|
+          yield time, record
+        end
       else
         raise "'#{EVENT_RECORD_PARAMETER}' parameter is required"
       end
@@ -573,6 +536,8 @@ module Fluent::Plugin
           params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)
         elsif /^application\/json/.match?(@content_type)
           params['json'] = @body
+        elsif /^application\/csp-report/.match?(@content_type)
+          params['json'] = @body
         elsif /^application\/msgpack/.match?(@content_type)
           params['msgpack'] = @body
         elsif /^application\/x-ndjson/.match?(@content_type)
@@ -580,7 +545,7 @@ module Fluent::Plugin
         end
         path_info = uri.path
 
-        if (@add_query_params) 
+        if (@add_query_params)
 
           query_params = WEBrick::HTTPUtils.parse_query(uri.query)
 

data/lib/fluent/plugin/in_tail.rb

@@ -52,6 +52,7 @@ module Fluent::Plugin
       super
       @paths = []
       @tails = {}
+      @tails_rotate_wait = {}
       @pf_file = nil
       @pf = nil
       @ignore_list = []
@@ -64,6 +65,8 @@ module Fluent::Plugin
     config_param :path, :string
     desc 'path delimiter used for spliting path config'
     config_param :path_delimiter, :string, default: ','
+    desc 'Choose using glob patterns. Adding capabilities to handle [] and ?, and {}.'
+    config_param :glob_policy, :enum, list: [:backward_compatible, :extended, :always], default: :backward_compatible
     desc 'The tag of the event.'
     config_param :tag, :string
     desc 'The paths to exclude the files from watcher list.'
@@ -140,6 +143,14 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "either of enable_watch_timer or enable_stat_watcher must be true"
       end
 
+      if @glob_policy == :always && @path_delimiter == ','
+        raise Fluent::ConfigError, "cannot use glob_policy as always with the default path_delimitor: `,\""
+      end
+
+      if @glob_policy == :extended && /\{.*,.*\}/.match(@path) && extended_glob_pattern(@path)
+        raise Fluent::ConfigError, "cannot include curly braces with glob patterns in `#{@path}\". Use glob_policy always instead."
+      end
+
       if RESERVED_CHARS.include?(@path_delimiter)
         rc = RESERVED_CHARS.join(', ')
         raise Fluent::ConfigError, "#{rc} are reserved words: #{@path_delimiter}"
@@ -267,6 +278,9 @@ module Fluent::Plugin
       @shutdown_start_time = Fluent::Clock.now
       # during shutdown phase, don't close io. It should be done in close after all threads are stopped. See close.
       stop_watchers(existence_path, immediate: true, remove_watcher: false)
+      @tails_rotate_wait.keys.each do |tw|
+        detach_watcher(tw, @tails_rotate_wait[tw][:ino], false)
+      end
       @pf_file.close if @pf_file
 
       super
@@ -275,6 +289,7 @@ module Fluent::Plugin
     def close
       super
       # close file handles after all threads stopped (in #close of thread plugin helper)
+      # It may be because we need to wait IOHanlder.ready_to_shutdown()
       close_watcher_handles
     end
 
@@ -283,6 +298,28 @@ module Fluent::Plugin
         @capability.have_capability?(:effective, :dac_override)
     end
 
+    def extended_glob_pattern(path)
+      path.include?('*') || path.include?('?') || /\[.*\]/.match(path)
+    end
+
+    # Curly braces is not supported with default path_delimiter
+    # because the default delimiter of path is ",".
+    # This should be collided for wildcard pattern for curly braces and
+    # be handled as an error on #configure.
+    def use_glob?(path)
+      if @glob_policy == :always
+        # For future extensions, we decided to use `always' term to handle
+        # regular expressions as much as possible.
+        # This is because not using `true' as a returning value
+        # when choosing :always here.
+        extended_glob_pattern(path) || /\{.*,.*\}/.match(path)
+      elsif @glob_policy == :extended
+        extended_glob_pattern(path)
+      elsif @glob_policy == :backward_compatible
+        path.include?('*')
+      end
+    end
+
     def expand_paths
       date = Fluent::EventTime.now
       paths = []
@@ -292,7 +329,7 @@ module Fluent::Plugin
                else
                  date.to_time.strftime(path)
                end
-        if path.include?('*')
+        if use_glob?(path)
           paths += Dir.glob(path).select { |p|
             begin
               is_file = !File.directory?(p)
@@ -327,7 +364,7 @@ module Fluent::Plugin
                else
                  date.to_time.strftime(path)
                end
-        path.include?('*') ? Dir.glob(path) : path
+        use_glob?(path) ? Dir.glob(path) : path
       }.flatten.uniq
       # filter out non existing files, so in case pattern is without '*' we don't do unnecessary work
       hash = {}
@@ -516,6 +553,9 @@ module Fluent::Plugin
           tw.close
         end
       end
+      @tails_rotate_wait.keys.each do |tw|
+        tw.close
+      end
     end
 
     # refresh_watchers calls @tails.keys so we don't use stop_watcher -> start_watcher sequence for safety.
@@ -570,10 +610,6 @@ module Fluent::Plugin
       detach_watcher_after_rotate_wait(tail_watcher, pe.read_inode)
     end
 
-    # TailWatcher#close is called by another thread at shutdown phase.
-    # It causes 'can't modify string; temporarily locked' error in IOHandler
-    # so adding close_io argument to avoid this problem.
-    # At shutdown, IOHandler's io will be released automatically after detached the event loop
     def detach_watcher(tw, ino, close_io = true)
       if @follow_inodes && tw.ino != ino
         log.warn("detach_watcher could be detaching an unexpected tail_watcher with a different ino.",
@@ -604,7 +640,11 @@ module Fluent::Plugin
       if @open_on_every_update
         # Detach now because it's already closed, waiting it doesn't make sense.
         detach_watcher(tw, ino)
-      elsif throttling_is_enabled?(tw)
+      end
+
+      return if @tails_rotate_wait[tw]
+
+      if throttling_is_enabled?(tw)
         # When the throttling feature is enabled, it might not reach EOF yet.
         # Should ensure to read all contents before closing it, with keeping throttling.
         start_time_to_wait = Fluent::Clock.now
@@ -612,14 +652,18 @@ module Fluent::Plugin
           elapsed = Fluent::Clock.now - start_time_to_wait
           if tw.eof? && elapsed >= @rotate_wait
             timer.detach
+            @tails_rotate_wait.delete(tw)
             detach_watcher(tw, ino)
           end
         end
+        @tails_rotate_wait[tw] = { ino: ino, timer: timer }
       else
         # when the throttling feature isn't enabled, just wait @rotate_wait
-        timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
+        timer = timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
+          @tails_rotate_wait.delete(tw)
           detach_watcher(tw, ino)
         end
+        @tails_rotate_wait[tw] = { ino: ino, timer: timer }
       end
     end