fluentd 1.13.3 → 1.16.5

data/lib/fluent/plugin/in_http.rb

@@ -74,6 +74,8 @@ module Fluent::Plugin
     config_param :blocking_timeout, :time, default: 0.5
     desc 'Set a allow list of domains that can do CORS (Cross-Origin Resource Sharing)'
     config_param :cors_allow_origins, :array, default: nil
+    desc 'Tells browsers whether to expose the response to frontend when the credentials mode is "include".'
+    config_param :cors_allow_credentials, :bool, default: false
     desc 'Respond with empty gif image of 1x1 pixel.'
     config_param :respond_with_empty_img, :bool, default: false
     desc 'Respond status code with 204.'
@@ -112,6 +114,12 @@ module Fluent::Plugin
 
       super
 
+      if @cors_allow_credentials
+        if @cors_allow_origins.nil? || @cors_allow_origins.include?('*')
+          raise Fluent::ConfigError, "Cannot enable cors_allow_credentials without specific origins"
+        end
+      end
+
       m = if @parser_configs.first['@type'] == 'in_http'
             @parser_msgpack = parser_create(usage: 'parser_in_http_msgpack', type: 'msgpack')
             @parser_msgpack.time_key = nil
@@ -279,7 +287,10 @@ module Fluent::Plugin
     private
 
     def on_server_connect(conn)
-      handler = Handler.new(conn, @km, method(:on_request), @body_size_limit, @format_name, log, @cors_allow_origins, @add_query_params)
+      handler = Handler.new(conn, @km, method(:on_request),
+                            @body_size_limit, @format_name, log,
+                            @cors_allow_origins, @cors_allow_credentials,
+                            @add_query_params)
 
       conn.on(:data) do |data|
         handler.on_read(data)
@@ -303,8 +314,16 @@ module Fluent::Plugin
         @parser_json.parse(js) do |_time, record|
           return nil, record
         end
+      elsif ndjson = params['ndjson']
+        events = []
+        ndjson.split(/\r?\n/).each do |js|
+          @parser_json.parse(js) do |_time, record|
+            events.push(record)
+          end
+        end
+        return nil, events
       else
-        raise "'json' or 'msgpack' parameter is required"
+        raise "'json', 'ndjson' or 'msgpack' parameter is required"
       end
     end
 
@@ -356,7 +375,8 @@ module Fluent::Plugin
     class Handler
       attr_reader :content_type
 
-      def initialize(io, km, callback, body_size_limit, format_name, log, cors_allow_origins, add_query_params)
+      def initialize(io, km, callback, body_size_limit, format_name, log,
+                     cors_allow_origins, cors_allow_credentials, add_query_params)
         @io = io
         @km = km
         @callback = callback
@@ -365,6 +385,7 @@ module Fluent::Plugin
         @format_name = format_name
         @log = log
         @cors_allow_origins = cors_allow_origins
+        @cors_allow_credentials = cors_allow_credentials
         @idle = 0
         @add_query_params = add_query_params
         @km.add(self)
@@ -407,7 +428,7 @@ module Fluent::Plugin
         @content_type = ""
         @content_encoding = ""
         headers.each_pair {|k,v|
-          @env["HTTP_#{k.gsub('-','_').upcase}"] = v
+          @env["HTTP_#{k.tr('-','_').upcase}"] = v
           case k
           when /\AExpect\z/i
             expect = v
@@ -418,9 +439,9 @@ module Fluent::Plugin
           when /\AContent-Encoding\Z/i
             @content_encoding = v
           when /\AConnection\Z/i
-            if v =~ /close/i
+            if /close/i.match?(v)
               @keep_alive = false
-            elsif v =~ /Keep-alive/i
+            elsif /Keep-alive/i.match?(v)
               @keep_alive = true
             end
           when /\AOrigin\Z/i
@@ -491,6 +512,9 @@ module Fluent::Plugin
           send_response_and_close(RES_200_STATUS, header, "")
         elsif include_cors_allow_origin
           header["Access-Control-Allow-Origin"] = @origin
+          if @cors_allow_credentials
+            header["Access-Control-Allow-Credentials"] = true
+          end
           send_response_and_close(RES_200_STATUS, header, "")
         else
           send_response_and_close(RES_403_STATUS, {}, "")
@@ -542,15 +566,17 @@ module Fluent::Plugin
 
         if @format_name != 'default'
           params[EVENT_RECORD_PARAMETER] = @body
-        elsif @content_type =~ /^application\/x-www-form-urlencoded/
+        elsif /^application\/x-www-form-urlencoded/.match?(@content_type)
           params.update WEBrick::HTTPUtils.parse_query(@body)
         elsif @content_type =~ /^multipart\/form-data; boundary=(.+)/
           boundary = WEBrick::HTTPUtils.dequote($1)
           params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)
-        elsif @content_type =~ /^application\/json/
+        elsif /^application\/json/.match?(@content_type)
           params['json'] = @body
-        elsif @content_type =~ /^application\/msgpack/
+        elsif /^application\/msgpack/.match?(@content_type)
           params['msgpack'] = @body
+        elsif /^application\/x-ndjson/.match?(@content_type)
+          params['ndjson'] = @body
         end
         path_info = uri.path
 
@@ -559,7 +585,7 @@ module Fluent::Plugin
           query_params = WEBrick::HTTPUtils.parse_query(uri.query)
 
           query_params.each_pair {|k,v|
-            params["QUERY_#{k.gsub('-','_').upcase}"] = v
+            params["QUERY_#{k.tr('-','_').upcase}"] = v
           }
         end
 
@@ -576,6 +602,9 @@ module Fluent::Plugin
             header['Access-Control-Allow-Origin'] = '*'
           elsif include_cors_allow_origin
             header['Access-Control-Allow-Origin'] = @origin
+            if @cors_allow_credentials
+              header["Access-Control-Allow-Credentials"] = true
+            end
           end
         end
 

data/lib/fluent/plugin/in_monitor_agent.rb

@@ -238,7 +238,7 @@ module Fluent::Plugin
       'buffer_queue_length' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.queue.size },
       'buffer_timekeys' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.timekeys },
       'buffer_total_queued_size' => ->(){ throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer); @buffer.stage_size + @buffer.queue_size },
-      'retry_count' => ->(){ instance_variable_defined?(:@num_errors) ? @num_errors : nil },
+      'retry_count' => ->(){ respond_to?(:num_errors) ? num_errors : nil },
     }
 
     def all_plugins
@@ -335,7 +335,9 @@ module Fluent::Plugin
       }
 
       if pe.respond_to?(:statistics)
-        obj.merge!(pe.statistics['output'] || {})
+        obj.merge!(pe.statistics.dig('output') || {})
+        obj.merge!(pe.statistics.dig('filter') || {})
+        obj.merge!(pe.statistics.dig('input') || {})
       end
 
       obj['retry'] = get_retry_info(pe.retry) if opts[:with_retry] && pe.instance_variable_defined?(:@retry)

data/lib/fluent/plugin/in_sample.rb

@@ -64,7 +64,7 @@ module Fluent::Plugin
     def configure(conf)
       super
       @sample_index = 0
-      config = conf.elements.select{|e| e.name == 'storage' }.first
+      config = conf.elements.find{|e| e.name == 'storage' }
       @storage = storage_create(usage: 'suspend', conf: config, default_type: DEFAULT_STORAGE_TYPE)
     end
 

data/lib/fluent/plugin/in_syslog.rb

@@ -89,6 +89,8 @@ module Fluent::Plugin
     config_param :source_hostname_key, :string, default: nil
     desc 'Try to resolve hostname from IP addresses or not.'
     config_param :resolve_hostname, :bool, default: nil
+    desc 'Check the remote connection is still available by sending a keepalive packet if this value is true.'
+    config_param :send_keepalive_packet, :bool, default: false
     desc 'The field name of source address of sender.'
     config_param :source_address_key, :string, default: nil
     desc 'The field name of the severity.'
@@ -143,6 +145,11 @@ module Fluent::Plugin
       end
 
       @_event_loop_run_timeout = @blocking_timeout
+
+      protocol = @protocol_type || @transport_config.protocol
+      if @send_keepalive_packet && protocol == :udp
+        raise Fluent::ConfigError, "send_keepalive_packet is available for tcp/tls"
+      end
     end
 
     def multi_workers_ready?
@@ -173,7 +180,12 @@ module Fluent::Plugin
 
       delimiter = octet_count_frame ? " " : @delimiter
       delimiter_size = delimiter.size
-      server_create_connection(tls ? :in_syslog_tls_server : :in_syslog_tcp_server, @port, bind: @bind, resolve_name: @resolve_hostname) do |conn|
+      server_create_connection(
+        tls ? :in_syslog_tls_server : :in_syslog_tcp_server, @port,
+        bind: @bind,
+        resolve_name: @resolve_hostname,
+        send_keepalive_packet: @send_keepalive_packet
+      ) do |conn|
         conn.data do |data|
           buffer = conn.buffer
           buffer << data

data/lib/fluent/plugin/in_tail/group_watch.rb

@@ -0,0 +1,204 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'fluent/plugin/input'
+
+module Fluent::Plugin
+  class TailInput < Fluent::Plugin::Input
+    module GroupWatchParams
+      include Fluent::Configurable
+
+      DEFAULT_KEY = /.*/
+      DEFAULT_LIMIT = -1
+      REGEXP_JOIN = "_"
+
+      config_section :group, param_name: :group, required: false, multi: false do
+        desc 'Regex for extracting group\'s metadata'
+        config_param :pattern,
+                     :regexp,
+                     default: /^\/var\/log\/containers\/(?<podname>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\/[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container>.+)-(?<docker_id>[a-z0-9]{64})\.log$/
+
+        desc 'Period of time in which the group_line_limit is applied'
+        config_param :rate_period, :time, default: 5
+
+        config_section :rule, param_name: :rule, required: true, multi: true do
+          desc 'Key-value pairs for grouping'
+          config_param :match, :hash, value_type: :regexp, default: { namespace: [DEFAULT_KEY], podname: [DEFAULT_KEY] }
+          desc 'Maximum number of log lines allowed per group over a period of rate_period'
+          config_param :limit, :integer, default: DEFAULT_LIMIT
+        end
+      end
+    end
+
+    module GroupWatch
+      def self.included(mod)
+        mod.include GroupWatchParams
+      end
+
+      attr_reader :group_watchers, :default_group_key
+
+      def initialize
+        super
+        @group_watchers = {}
+        @group_keys = nil
+        @default_group_key = nil
+      end
+
+      def configure(conf)
+        super
+
+        unless @group.nil?
+          ## Ensuring correct time period syntax
+          @group.rule.each { |rule|
+            raise "Metadata Group Limit >= DEFAULT_LIMIT" unless rule.limit >= GroupWatchParams::DEFAULT_LIMIT
+          }
+
+          @group_keys = Regexp.compile(@group.pattern).named_captures.keys
+          @default_group_key = ([GroupWatchParams::DEFAULT_KEY] * @group_keys.length).join(GroupWatchParams::REGEXP_JOIN)
+
+          ## Ensures that "specific" rules (with larger number of `rule.match` keys)
+          ## have a higher priority against "generic" rules (with less number of `rule.match` keys).
+          ## This will be helpful when a file satisfies more than one rule.
+          @group.rule.sort_by! { |rule| -rule.match.length() }
+          construct_groupwatchers
+          @group_watchers[@default_group_key] ||= GroupWatcher.new(@group.rate_period, GroupWatchParams::DEFAULT_LIMIT)
+        end
+      end
+
+      def add_path_to_group_watcher(path)
+        return nil if @group.nil?
+        group_watcher = find_group_from_metadata(path)
+        group_watcher.add(path) unless group_watcher.include?(path)
+        group_watcher
+      end
+
+      def remove_path_from_group_watcher(path)
+        return if @group.nil?
+        group_watcher = find_group_from_metadata(path)
+        group_watcher.delete(path)
+      end
+
+      def construct_group_key(named_captures)
+        match_rule = []
+        @group_keys.each { |key|
+          match_rule.append(named_captures.fetch(key, GroupWatchParams::DEFAULT_KEY))
+        }
+        match_rule = match_rule.join(GroupWatchParams::REGEXP_JOIN)
+
+        match_rule
+      end
+
+      def construct_groupwatchers
+        @group.rule.each { |rule|
+          match_rule = construct_group_key(rule.match)
+          @group_watchers[match_rule] ||= GroupWatcher.new(@group.rate_period, rule.limit)
+        }
+      end
+
+      def find_group(metadata)
+        metadata_key = construct_group_key(metadata)
+        gw_key = @group_watchers.keys.find { |regexp| metadata_key.match?(regexp) && regexp != @default_group_key }
+        gw_key ||= @default_group_key
+
+        @group_watchers[gw_key]
+      end
+
+      def find_group_from_metadata(path)
+        begin
+          metadata = @group.pattern.match(path).named_captures
+          group_watcher = find_group(metadata)
+        rescue
+          log.warn "Cannot find group from metadata, Adding file in the default group"
+          group_watcher = @group_watchers[@default_group_key]
+        end
+
+        group_watcher
+      end
+    end
+
+    class GroupWatcher
+      attr_accessor :current_paths, :limit, :number_lines_read, :start_reading_time, :rate_period
+
+      FileCounter = Struct.new(
+        :number_lines_read,
+        :start_reading_time,
+      )
+
+      def initialize(rate_period = 60, limit = -1)
+        @current_paths = {}
+        @rate_period = rate_period
+        @limit = limit
+      end
+
+      def add(path)
+        @current_paths[path] = FileCounter.new(0, nil)
+      end
+
+      def include?(path)
+        @current_paths.key?(path)
+      end
+
+      def size
+        @current_paths.size
+      end
+
+      def delete(path)
+        @current_paths.delete(path)
+      end
+
+      def update_reading_time(path)
+        @current_paths[path].start_reading_time ||= Fluent::Clock.now
+      end
+
+      def update_lines_read(path, value)
+        @current_paths[path].number_lines_read += value
+      end
+
+      def reset_counter(path)
+        @current_paths[path].start_reading_time = nil
+        @current_paths[path].number_lines_read = 0
+      end
+
+      def time_spent_reading(path)
+        Fluent::Clock.now - @current_paths[path].start_reading_time
+      end
+
+      def limit_time_period_reached?(path)
+        time_spent_reading(path) < @rate_period
+      end
+
+      def limit_lines_reached?(path)
+        return true unless include?(path)
+        return true if @limit == 0
+
+        return false if @limit < 0
+        return false if @current_paths[path].number_lines_read < @limit / size
+
+        # update_reading_time(path)
+        if limit_time_period_reached?(path) # Exceeds limit
+          true
+        else # Does not exceed limit
+          reset_counter(path)
+          false
+        end
+      end
+
+      def to_s
+        super + " current_paths: #{@current_paths} rate_period: #{@rate_period} limit: #{@limit}"
+      end
+    end
+  end
+end

data/lib/fluent/plugin/in_tail/position_file.rb

@@ -53,10 +53,16 @@ module Fluent::Plugin
         }
       end
 
+      def unwatch_removed_targets(existing_targets)
+        @map.reject { |key, entry|
+          existing_targets.key?(key)
+        }.each_key { |key|
+          unwatch_key(key)
+        }
+      end
+
       def unwatch(target_info)
-        if (entry = @map.delete(@follow_inodes ? target_info.ino : target_info.path))
-          entry.update_pos(UNWATCHED_POSITION)
-        end
+        unwatch_key(@follow_inodes ? target_info.ino : target_info.path)
       end
 
       def load(existing_targets = nil)
@@ -96,6 +102,7 @@ module Fluent::Plugin
         end
 
         entries = fetch_compacted_entries
+        @logger&.debug "Compacted entries: ", entries.keys
 
         @file_mutex.synchronize do
           if last_modified == @file.mtime && size == @file.size
@@ -117,17 +124,31 @@ module Fluent::Plugin
 
       private
 
+      def unwatch_key(key)
+        if (entry = @map.delete(key))
+          entry.update_pos(UNWATCHED_POSITION)
+        end
+      end
+
       def compact(existing_targets = nil)
         @file_mutex.synchronize do
-          entries = fetch_compacted_entries(existing_targets).values.map(&:to_entry_fmt)
+          entries = fetch_compacted_entries
+          @logger&.debug "Compacted entries: ", entries.keys
+
+          if existing_targets
+            entries = remove_deleted_files_entries(entries, existing_targets)
+            @logger&.debug "Remove missing entries.",
+                           existing_targets: existing_targets.keys,
+                           entries_after_removing: entries.keys
+          end
 
           @file.pos = 0
           @file.truncate(0)
-          @file.write(entries.join)
+          @file.write(entries.values.map(&:to_entry_fmt).join)
         end
       end
 
-      def fetch_compacted_entries(existing_targets = nil)
+      def fetch_compacted_entries
         entries = {}
 
         @file.pos = 0
@@ -145,31 +166,24 @@ module Fluent::Plugin
           if pos == UNWATCHED_POSITION
             @logger.debug "Remove unwatched line from pos_file: #{line}" if @logger
           else
-            if entries.include?(path)
-              @logger.warn("#{path} already exists. use latest one: deleted #{entries[path]}") if @logger
-            end
-
             if @follow_inodes
+              @logger&.warn("#{path} (inode: #{ino}) already exists. use latest one: deleted #{entries[ino]}") if entries.include?(ino)
               entries[ino] = Entry.new(path, pos, ino, file_pos + path.bytesize + 1)
             else
+              @logger&.warn("#{path} already exists. use latest one: deleted #{entries[path]}") if entries.include?(path)
               entries[path] = Entry.new(path, pos, ino, file_pos + path.bytesize + 1)
             end
             file_pos += line.size
           end
         end
 
-        entries = remove_deleted_files_entries(entries, existing_targets) if @follow_inodes
         entries
       end
 
       def remove_deleted_files_entries(existent_entries, existing_targets)
-        if existing_targets
-          existent_entries.select { |path_or_ino|
-            existing_targets.key?(path_or_ino)
-          }
-        else
-          existent_entries
-        end
+        existent_entries.select { |path_or_ino|
+          existing_targets.key?(path_or_ino)
+        }
       end
     end
 
@@ -250,20 +264,6 @@ module Fluent::Plugin
       end
     end
 
-    TargetInfo = Struct.new(:path, :ino) do
-      def ==(other)
-        return false unless other.is_a?(TargetInfo)
-        self.path == other.path
-      end
-
-      def hash
-        self.path.hash
-      end
-
-      def eql?(other)
-        return false unless other.is_a?(TargetInfo)
-        self.path == other.path
-      end
-    end
+    TargetInfo = Struct.new(:path, :ino)
   end
 end