fluentd 1.14.4 → 1.15.0

data/lib/fluent/plugin/in_tail/group_watch.rb

@@ -0,0 +1,204 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/input'
+
+module Fluent::Plugin
+  class TailInput < Fluent::Plugin::Input
+    module GroupWatchParams
+      include Fluent::Configurable
+
+      DEFAULT_KEY = /.*/
+      DEFAULT_LIMIT = -1
+      REGEXP_JOIN = "_"
+
+      config_section :group, param_name: :group, required: false, multi: false do
+        desc 'Regex for extracting group\'s metadata'
+        config_param :pattern,
+                     :regexp,
+                     default: /^\/var\/log\/containers\/(?<podname>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\/[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container>.+)-(?<docker_id>[a-z0-9]{64})\.log$/
+
+        desc 'Period of time in which the group_line_limit is applied'
+        config_param :rate_period, :time, default: 5
+
+        config_section :rule, param_name: :rule, required: true, multi: true do
+          desc 'Key-value pairs for grouping'
+          config_param :match, :hash, value_type: :regexp, default: { namespace: [DEFAULT_KEY], podname: [DEFAULT_KEY] }
+          desc 'Maximum number of log lines allowed per group over a period of rate_period'
+          config_param :limit, :integer, default: DEFAULT_LIMIT
+        end
+      end
+    end
+
+    module GroupWatch
+      def self.included(mod)
+        mod.include GroupWatchParams
+      end
+
+      attr_reader :group_watchers, :default_group_key
+
+      def initialize
+        super
+        @group_watchers = {}
+        @group_keys = nil
+        @default_group_key = nil
+      end
+
+      def configure(conf)
+        super
+
+        unless @group.nil?
+          ## Ensuring correct time period syntax
+          @group.rule.each { |rule|
+            raise "Metadata Group Limit >= DEFAULT_LIMIT" unless rule.limit >= GroupWatchParams::DEFAULT_LIMIT
+          }
+
+          @group_keys = Regexp.compile(@group.pattern).named_captures.keys
+          @default_group_key = ([GroupWatchParams::DEFAULT_KEY] * @group_keys.length).join(GroupWatchParams::REGEXP_JOIN)
+
+          ## Ensures that "specific" rules (with larger number of `rule.match` keys)
+          ## have a higher priority against "generic" rules (with less number of `rule.match` keys).
+          ## This will be helpful when a file satisfies more than one rule.
+          @group.rule.sort_by! { |rule| -rule.match.length() }
+          construct_groupwatchers
+          @group_watchers[@default_group_key] ||= GroupWatcher.new(@group.rate_period, GroupWatchParams::DEFAULT_LIMIT)
+        end
+      end
+
+      def add_path_to_group_watcher(path)
+        return nil if @group.nil?
+        group_watcher = find_group_from_metadata(path)
+        group_watcher.add(path) unless group_watcher.include?(path)
+        group_watcher
+      end
+
+      def remove_path_from_group_watcher(path)
+        return if @group.nil?
+        group_watcher = find_group_from_metadata(path)
+        group_watcher.delete(path)
+      end
+
+      def construct_group_key(named_captures)
+        match_rule = []
+        @group_keys.each { |key|
+          match_rule.append(named_captures.fetch(key, GroupWatchParams::DEFAULT_KEY))
+        }
+        match_rule = match_rule.join(GroupWatchParams::REGEXP_JOIN)
+
+        match_rule
+      end
+
+      def construct_groupwatchers
+        @group.rule.each { |rule|
+          match_rule = construct_group_key(rule.match)
+          @group_watchers[match_rule] ||= GroupWatcher.new(@group.rate_period, rule.limit)
+        }
+      end
+
+      def find_group(metadata)
+        metadata_key = construct_group_key(metadata)
+        gw_key = @group_watchers.keys.find { |regexp| metadata_key.match?(regexp) && regexp != @default_group_key }
+        gw_key ||= @default_group_key
+
+        @group_watchers[gw_key]
+      end
+
+      def find_group_from_metadata(path)
+        begin
+          metadata = @group.pattern.match(path).named_captures
+          group_watcher = find_group(metadata)
+        rescue
+          log.warn "Cannot find group from metadata, Adding file in the default group"
+          group_watcher = @group_watchers[@default_group_key]
+        end
+
+        group_watcher
+      end
+    end
+
+    class GroupWatcher
+      attr_accessor :current_paths, :limit, :number_lines_read, :start_reading_time, :rate_period
+
+      FileCounter = Struct.new(
+        :number_lines_read,
+        :start_reading_time,
+      )
+
+      def initialize(rate_period = 60, limit = -1)
+        @current_paths = {}
+        @rate_period = rate_period
+        @limit = limit
+      end
+
+      def add(path)
+        @current_paths[path] = FileCounter.new(0, nil)
+      end
+
+      def include?(path)
+        @current_paths.key?(path)
+      end
+
+      def size
+        @current_paths.size
+      end
+
+      def delete(path)
+        @current_paths.delete(path)
+      end
+
+      def update_reading_time(path)
+        @current_paths[path].start_reading_time ||= Fluent::Clock.now
+      end
+
+      def update_lines_read(path, value)
+        @current_paths[path].number_lines_read += value
+      end
+
+      def reset_counter(path)
+        @current_paths[path].start_reading_time = nil
+        @current_paths[path].number_lines_read = 0
+      end
+
+      def time_spent_reading(path)
+        Fluent::Clock.now - @current_paths[path].start_reading_time
+      end
+
+      def limit_time_period_reached?(path)
+        time_spent_reading(path) < @rate_period
+      end
+
+      def limit_lines_reached?(path)
+        return true unless include?(path)
+        return true if @limit == 0
+
+        return false if @limit < 0
+        return false if @current_paths[path].number_lines_read < @limit / size
+
+        # update_reading_time(path)
+        if limit_time_period_reached?(path) # Exceeds limit
+          true
+        else # Does not exceed limit
+          reset_counter(path)
+          false
+        end
+      end
+
+      def to_s
+        super + " current_paths: #{@current_paths} rate_period: #{@rate_period} limit: #{@limit}"
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_tail/position_file.rb

@@ -250,20 +250,6 @@ module Fluent::Plugin
       end
     end
 
-    TargetInfo = Struct.new(:path, :ino) do
-      def ==(other)
-        return false unless other.is_a?(TargetInfo)
-        self.path == other.path
-      end
-
-      def hash
-        self.path.hash
-      end
-
-      def eql?(other)
-        return false unless other.is_a?(TargetInfo)
-        self.path == other.path
-      end
-    end
+    TargetInfo = Struct.new(:path, :ino)
   end
 end

data/lib/fluent/plugin/in_tail.rb

@@ -24,6 +24,7 @@ require 'fluent/plugin/parser_multiline'
 require 'fluent/variable_store'
 require 'fluent/capability'
 require 'fluent/plugin/in_tail/position_file'
+require 'fluent/plugin/in_tail/group_watch'
 
 if Fluent.windows?
   require_relative 'file_wrapper'
@@ -33,6 +34,8 @@ end
 
 module Fluent::Plugin
   class TailInput < Fluent::Plugin::Input
+    include GroupWatch
+
     Fluent::Plugin.register_input('tail', self)
 
     helpers :timer, :event_loop, :parser, :compat_parameters
@@ -354,11 +357,11 @@ module Fluent::Plugin
 
     def existence_path
       hash = {}
-      @tails.each_key {|target_info|
+      @tails.each {|path, tw|
         if @follow_inodes
-          hash[target_info.ino] = target_info
+          hash[tw.ino] = TargetInfo.new(tw.path, tw.ino)
         else
-          hash[target_info.path] = target_info
+          hash[tw.path] = TargetInfo.new(tw.path, tw.ino)
         end
       }
       hash
@@ -406,6 +409,8 @@ module Fluent::Plugin
         event_loop_attach(watcher)
       end
 
+      tw.group_watcher = add_path_to_group_watcher(target_info.path)
+
       tw
     rescue => e
       if tw
@@ -420,36 +425,31 @@ module Fluent::Plugin
     end
 
     def construct_watcher(target_info)
+      path = target_info.path
+
+      # The file might be rotated or removed after collecting paths, so check inode again here.
+      begin
+        target_info.ino = Fluent::FileWrapper.stat(path).ino
+      rescue Errno::ENOENT, Errno::EACCES
+        $log.warn "stat() for #{path} failed. Continuing without tailing it."
+        return
+      end
+
       pe = nil
       if @pf
         pe = @pf[target_info]
-        if @read_from_head && pe.read_inode.zero?
-          begin
-            pe.update(Fluent::FileWrapper.stat(target_info.path).ino, 0)
-          rescue Errno::ENOENT, Errno::EACCES
-            $log.warn "stat() for #{target_info.path} failed. Continuing without tailing it."
-          end
-        end
+        pe.update(target_info.ino, 0) if @read_from_head && pe.read_inode.zero?
       end
 
       begin
         tw = setup_watcher(target_info, pe)
       rescue WatcherSetupError => e
-        log.warn "Skip #{target_info.path} because unexpected setup error happens: #{e}"
+        log.warn "Skip #{path} because unexpected setup error happens: #{e}"
         return
       end
 
-      begin
-        target_info = TargetInfo.new(target_info.path, Fluent::FileWrapper.stat(target_info.path).ino)
-        @tails.delete(target_info)
-        @tails[target_info] = tw
-        tw.on_notify
-      rescue Errno::ENOENT, Errno::EACCES => e
-        $log.warn "stat() for #{target_info.path} failed with #{e.class.name}. Drop tail watcher for now."
-        # explicitly detach and unwatch watcher `tw`.
-        tw.unwatched = true
-        detach_watcher(tw, target_info.ino, false)
-      end
+      @tails[path] = tw
+      tw.on_notify
     end
 
     def start_watchers(targets_info)
@@ -461,10 +461,12 @@ module Fluent::Plugin
 
     def stop_watchers(targets_info, immediate: false, unwatched: false, remove_watcher: true)
       targets_info.each_value { |target_info|
+        remove_path_from_group_watcher(target_info.path)
+
         if remove_watcher
-          tw = @tails.delete(target_info)
+          tw = @tails.delete(target_info.path)
         else
-          tw = @tails[target_info]
+          tw = @tails[target_info.path]
         end
         if tw
           tw.unwatched = unwatched
@@ -478,8 +480,8 @@ module Fluent::Plugin
     end
 
     def close_watcher_handles
-      @tails.keys.each do |target_info|
-        tw = @tails.delete(target_info)
+      @tails.keys.each do |path|
+        tw = @tails.delete(path)
         if tw
           tw.close
         end
@@ -488,20 +490,20 @@ module Fluent::Plugin
 
     # refresh_watchers calls @tails.keys so we don't use stop_watcher -> start_watcher sequence for safety.
     def update_watcher(target_info, pe)
-      log.info("detected rotation of #{target_info.path}; waiting #{@rotate_wait} seconds")
+      path = target_info.path
+
+      log.info("detected rotation of #{path}; waiting #{@rotate_wait} seconds")
 
       if @pf
         pe_inode = pe.read_inode
-        target_info_from_position_entry = TargetInfo.new(target_info.path, pe_inode)
+        target_info_from_position_entry = TargetInfo.new(path, pe_inode)
         unless pe_inode == @pf[target_info_from_position_entry].read_inode
           log.debug "Skip update_watcher because watcher has been already updated by other inotify event"
           return
         end
       end
 
-      rotated_target_info = TargetInfo.new(target_info.path, pe.read_inode)
-      rotated_tw = @tails[rotated_target_info]
-      new_target_info = target_info.dup
+      rotated_tw = @tails[path]
 
       if @follow_inodes
         new_position_entry = @pf[target_info]
@@ -509,17 +511,13 @@ module Fluent::Plugin
         if new_position_entry.read_inode == 0
           # When follow_inodes is true, it's not cleaned up by refresh_watcher.
           # So it should be unwatched here explicitly.
-          rotated_tw.unwatched = true
-          # Make sure to delete old key, it has a different ino while the hash key is same.
-          @tails.delete(rotated_target_info)
-          @tails[new_target_info] = setup_watcher(new_target_info, new_position_entry)
-          @tails[new_target_info].on_notify
+          rotated_tw.unwatched = true if rotated_tw
+          @tails[path] = setup_watcher(target_info, new_position_entry)
+          @tails[path].on_notify
         end
       else
-        # Make sure to delete old key, it has a different ino while the hash key is same.
-        @tails.delete(rotated_target_info)
-        @tails[new_target_info] = setup_watcher(new_target_info, pe)
-        @tails[new_target_info].on_notify
+        @tails[path] = setup_watcher(target_info, pe)
+        @tails[path].on_notify
       end
       detach_watcher_after_rotate_wait(rotated_tw, pe.read_inode) if rotated_tw
     end
@@ -542,18 +540,19 @@ module Fluent::Plugin
       end
     end
 
+    def throttling_is_enabled?(tw)
+      return true if @read_bytes_limit_per_second > 0
+      return true if tw.group_watcher && tw.group_watcher.limit >= 0
+      false
+    end
+
     def detach_watcher_after_rotate_wait(tw, ino)
       # Call event_loop_attach/event_loop_detach is high-cost for short-live object.
       # If this has a problem with large number of files, use @_event_loop directly instead of timer_execute.
       if @open_on_every_update
         # Detach now because it's already closed, waiting it doesn't make sense.
         detach_watcher(tw, ino)
-      elsif @read_bytes_limit_per_second < 0
-        # throttling isn't enabled, just wait @rotate_wait
-        timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
-          detach_watcher(tw, ino)
-        end
-      else
+      elsif throttling_is_enabled?(tw)
         # When the throttling feature is enabled, it might not reach EOF yet.
         # Should ensure to read all contents before closing it, with keeping throttling.
         start_time_to_wait = Fluent::Clock.now
@@ -564,6 +563,11 @@ module Fluent::Plugin
             detach_watcher(tw, ino)
           end
         end
+      else
+        # when the throttling feature isn't enabled, just wait @rotate_wait
+        timer_execute(:in_tail_close_watcher, @rotate_wait, repeat: false) do
+          detach_watcher(tw, ino)
+        end
       end
     end
 
@@ -775,6 +779,7 @@ module Fluent::Plugin
       attr_reader :line_buffer_timer_flusher
       attr_accessor :unwatched  # This is used for removing position entry from PositionFile
       attr_reader :watchers
+      attr_accessor :group_watcher
 
       def tag
         @parsed_tag ||= @path.tr('/', '.').gsub(/\.+/, '.').gsub(/^\./, '')
@@ -997,6 +1002,10 @@ module Fluent::Plugin
           @log.info "following tail of #{@path}"
         end
 
+        def group_watcher
+          @watcher.group_watcher
+        end
+
         def on_notify
           @notify_mutex.synchronize { handle_notify }
         end
@@ -1054,6 +1063,7 @@ module Fluent::Plugin
 
         def handle_notify
           return if limit_bytes_per_second_reached?
+          return if group_watcher&.limit_lines_reached?(@path)
 
           with_io do |io|
             begin
@@ -1063,17 +1073,26 @@ module Fluent::Plugin
                 begin
                   while true
                     @start_reading_time ||= Fluent::Clock.now
+                    group_watcher&.update_reading_time(@path)
+
                     data = io.readpartial(BYTES_TO_READ, @iobuf)
                     @eof = false
                     @number_bytes_read += data.bytesize
                     @fifo << data
+
+                    n_lines_before_read = @lines.size
                     @fifo.read_lines(@lines)
+                    group_watcher&.update_lines_read(@path, @lines.size - n_lines_before_read)
+
+                    group_watcher_limit = group_watcher&.limit_lines_reached?(@path)
+                    @log.debug "Reading Limit exceeded #{@path} #{group_watcher.number_lines_read}" if group_watcher_limit
 
-                    if limit_bytes_per_second_reached? || should_shutdown_now?
+                    if group_watcher_limit || limit_bytes_per_second_reached? || should_shutdown_now?
                       # Just get out from tailing loop.
                       read_more = false
                       break
                     end
+
                     if @lines.size >= @read_lines_limit
                       # not to use too much memory in case the file is very large
                       read_more = true

data/lib/fluent/plugin/out_forward/socket_cache.rb

@@ -50,6 +50,7 @@ module Fluent::Plugin
       def checkin(sock)
         @mutex.synchronize do
           if (s = @inflight_sockets.delete(sock))
+            s.timeout = timeout
             @available_sockets[s.key] << s
           else
             @log.debug("there is no socket #{sock}")
@@ -122,6 +123,7 @@ module Fluent::Plugin
         t = Time.now
         if (s = @available_sockets[key].find { |sock| !expired_socket?(sock, time: t) })
           @inflight_sockets[s.sock] = @available_sockets[key].delete(s)
+          s.timeout = timeout
           s
         else
           nil

data/lib/fluent/plugin/output.rb

@@ -235,6 +235,7 @@ module Fluent
         @dequeued_chunks_mutex = nil
         @output_enqueue_thread = nil
         @output_flush_threads = nil
+        @output_flush_thread_current_position = 0
 
         @simple_chunking = nil
         @chunk_keys = @chunk_key_accessors = @chunk_key_time = @chunk_key_tag = nil
@@ -273,7 +274,7 @@ module Fluent
         super
 
         @num_errors_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "num_errors", help_text: "Number of count num errors")
-        @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of count emits")
+        @emit_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_count", help_text: "Number of count emits")
         @emit_records_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_records", help_text: "Number of emit records")
         @emit_size_metrics =  metrics_create(namespace: "fluentd", subsystem: "output", name: "emit_size", help_text: "Total size of emit events")
         @write_count_metrics = metrics_create(namespace: "fluentd", subsystem: "output", name: "write_count", help_text: "Number of writing events")
@@ -492,6 +493,7 @@ module Fluent
           @dequeued_chunks = []
           @dequeued_chunks_mutex = Mutex.new
 
+          @output_flush_thread_current_position = 0
           @buffer_config.flush_thread_count.times do |i|
             thread_title = "flush_thread_#{i}".to_sym
             thread_state = FlushThreadState.new(nil, nil, Mutex.new, ConditionVariable.new)
@@ -503,7 +505,6 @@ module Fluent
               @output_flush_threads << thread_state
             end
           end
-          @output_flush_thread_current_position = 0
 
           if !@under_plugin_development && (@flush_mode == :interval || @chunk_key_time)
             @output_enqueue_thread = thread_create(:enqueue_thread, &method(:enqueue_thread_run))
@@ -1275,48 +1276,57 @@ module Fluent
 
           unless @retry
             @retry = retry_state(@buffer_config.retry_randomize)
-            if error
-              log.warn "failed to flush the buffer.", retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
-              log.warn_backtrace error.backtrace
+
+            if @retry.limit?
+              handle_limit_reached(error)
+            elsif error
+              log_retry_error(error, chunk_id_hex, using_secondary)
             end
+
             return
           end
 
           # @retry exists
 
-          if @retry.limit?
-            if error
-              records = @buffer.queued_records
-              msg = "failed to flush the buffer, and hit limit for retries. dropping all chunks in the buffer queue."
-              log.error msg, retry_times: @retry.steps, records: records, error: error
-              log.error_backtrace error.backtrace
-            end
-            @buffer.clear_queue!
-            log.debug "buffer queue cleared"
-            @retry = nil
+          # Ensure that the current time is greater than or equal to @retry.next_time to avoid the situation when
+          # @retry.step is called almost as many times as the number of flush threads in a short time.
+          if Time.now >= @retry.next_time
+            @retry.step
           else
-            # Ensure that the current time is greater than or equal to @retry.next_time to avoid the situation when
-            # @retry.step is called almost as many times as the number of flush threads in a short time.
-            if Time.now >= @retry.next_time
-              @retry.step
-            else
-              @retry.recalc_next_time # to prevent all flush threads from retrying at the same time
-            end
-            if error
-              if using_secondary
-                msg = "failed to flush the buffer with secondary output."
-                log.warn msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
-                log.warn_backtrace error.backtrace
-              else
-                msg = "failed to flush the buffer."
-                log.warn msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error
-                log.warn_backtrace error.backtrace
-              end
-            end
+            @retry.recalc_next_time # to prevent all flush threads from retrying at the same time
+          end
+
+          if @retry.limit?
+            handle_limit_reached(error)
+          elsif error
+            log_retry_error(error, chunk_id_hex, using_secondary)
           end
         end
       end
 
+      def log_retry_error(error, chunk_id_hex, using_secondary)
+        return unless error
+        if using_secondary
+          msg = "failed to flush the buffer with secondary output."
+        else
+          msg = "failed to flush the buffer."
+        end
+        log.warn(msg, retry_times: @retry.steps, next_retry_time: @retry.next_time.round, chunk: chunk_id_hex, error: error)
+        log.warn_backtrace(error.backtrace)
+      end
+
+      def handle_limit_reached(error)
+        if error
+          records = @buffer.queued_records
+          msg = "Hit limit for retries. dropping all chunks in the buffer queue."
+          log.error msg, retry_times: @retry.steps, records: records, error: error
+          log.error_backtrace error.backtrace
+        end
+        @buffer.clear_queue!
+        log.debug "buffer queue cleared"
+        @retry = nil
+      end
+
       def retry_state(randomize)
         if @secondary
           retry_state_create(

data/lib/fluent/plugin/parser.rb

@@ -89,7 +89,7 @@ module Fluent
       #      : format[, timezone]
 
       config_param :time_key, :string, default: nil
-      config_param :null_value_pattern, :string, default: nil
+      config_param :null_value_pattern, :regexp, default: nil
       config_param :null_empty_string, :bool, default: false
       config_param :estimate_current_event, :bool, default: true
       config_param :keep_time_key, :bool, default: false
@@ -115,9 +115,8 @@ module Fluent
         super
 
         @time_parser = time_parser_create
-        @null_value_regexp = @null_value_pattern && Regexp.new(@null_value_pattern)
         @type_converters = build_type_converters(@types)
-        @execute_convert_values = @type_converters || @null_value_regexp || @null_empty_string
+        @execute_convert_values = @type_converters || @null_value_pattern || @null_empty_string
         @timeout_checker = if @timeout
                              class << self
                                alias_method :parse_orig, :parse
@@ -220,7 +219,7 @@ module Fluent
         return time, record
       end
 
-      def string_like_null(value, null_empty_string = @null_empty_string, null_value_regexp = @null_value_regexp)
+      def string_like_null(value, null_empty_string = @null_empty_string, null_value_regexp = @null_value_pattern)
         null_empty_string && value.empty? || null_value_regexp && string_safe_encoding(value){|s| null_value_regexp.match(s) }
       end
 

data/lib/fluent/plugin/parser_syslog.rb

@@ -484,7 +484,7 @@ module Fluent
 
         time = begin
                  @time_parser_rfc5424.parse(time_str)
-               rescue Fluent::TimeParser::TimeParseError => e
+               rescue Fluent::TimeParser::TimeParseError
                  @time_parser_rfc5424_without_subseconds.parse(time_str)
                end
         record['time'] = time_str if @keep_time_key