fluentd 1.16.4-x64-mingw32 → 1.17.0-x64-mingw32

data/lib/fluent/plugin/in_http.rb

@@ -203,54 +203,24 @@ module Fluent::Plugin
       begin
         path = path_info[1..-1]  # remove /
         tag = path.split('/').join('.')
-        record_time, record = parse_params(params)
 
-        # Skip nil record
-        if record.nil?
-          log.debug { "incoming event is invalid: path=#{path_info} params=#{params.to_json}" }
-          if @respond_with_empty_img
-            return RESPONSE_IMG
-          else
-            if @use_204_response
-              return RESPONSE_204
-            else
-              return RESPONSE_200
-            end
+        mes = Fluent::MultiEventStream.new
+        parse_params(params) do |record_time, record|
+          if record.nil?
+            log.debug { "incoming event is invalid: path=#{path_info} params=#{params.to_json}" }
+            next
           end
-        end
 
-        mes = nil
-        # Support batched requests
-        if record.is_a?(Array)
-          mes = Fluent::MultiEventStream.new
-          record.each do |single_record|
-            add_params_to_record(single_record, params)
-
-            if param_time = params['time']
-              param_time = param_time.to_f
-              single_time = param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
-            elsif @custom_parser
-              single_time = @custom_parser.parse_time(single_record)
-              single_time, single_record = @custom_parser.convert_values(single_time, single_record)
-            else
-              single_time = convert_time_field(single_record)
-            end
-
-            mes.add(single_time, single_record)
-          end
-        else
           add_params_to_record(record, params)
 
           time = if param_time = params['time']
                    param_time = param_time.to_f
                    param_time.zero? ? Fluent::EventTime.now : @float_time_parser.parse(param_time)
                  else
-                   if record_time.nil?
-                     convert_time_field(record)
-                   else
-                     record_time
-                   end
+                   record_time.nil? ? convert_time_field(record) : record_time
                  end
+
+          mes.add(time, record)
         end
       rescue => e
         if @dump_error_log
@@ -261,11 +231,7 @@ module Fluent::Plugin
 
       # TODO server error
       begin
-        if mes
-          router.emit_stream(tag, mes)
-        else
-          router.emit(tag, time, record)
-        end
+        router.emit_stream(tag, mes) unless mes.empty?
       rescue => e
         if @dump_error_log
           log.error "failed to emit data", error: e
@@ -308,20 +274,18 @@ module Fluent::Plugin
     def parse_params_default(params)
       if msgpack = params['msgpack']
         @parser_msgpack.parse(msgpack) do |_time, record|
-          return nil, record
+          yield nil, record
         end
       elsif js = params['json']
         @parser_json.parse(js) do |_time, record|
-          return nil, record
+          yield nil, record
         end
       elsif ndjson = params['ndjson']
-        events = []
         ndjson.split(/\r?\n/).each do |js|
           @parser_json.parse(js) do |_time, record|
-            events.push(record)
+            yield nil, record
           end
         end
-        return nil, events
       else
         raise "'json', 'ndjson' or 'msgpack' parameter is required"
       end
@@ -329,10 +293,9 @@ module Fluent::Plugin
 
     def parse_params_with_parser(params)
       if content = params[EVENT_RECORD_PARAMETER]
-        @custom_parser.parse(content) { |time, record|
-          raise "Received event is not #{@format_name}: #{content}" if record.nil?
-          return time, record
-        }
+        @custom_parser.parse(content) do |time, record|
+          yield time, record
+        end
       else
         raise "'#{EVENT_RECORD_PARAMETER}' parameter is required"
       end
@@ -573,6 +536,8 @@ module Fluent::Plugin
           params.update WEBrick::HTTPUtils.parse_form_data(@body, boundary)
         elsif /^application\/json/.match?(@content_type)
           params['json'] = @body
+        elsif /^application\/csp-report/.match?(@content_type)
+          params['json'] = @body
         elsif /^application\/msgpack/.match?(@content_type)
           params['msgpack'] = @body
         elsif /^application\/x-ndjson/.match?(@content_type)
@@ -580,7 +545,7 @@ module Fluent::Plugin
         end
         path_info = uri.path
 
-        if (@add_query_params) 
+        if (@add_query_params)
 
           query_params = WEBrick::HTTPUtils.parse_query(uri.query)
 

data/lib/fluent/plugin/in_tail.rb

@@ -65,6 +65,8 @@ module Fluent::Plugin
     config_param :path, :string
     desc 'path delimiter used for spliting path config'
     config_param :path_delimiter, :string, default: ','
+    desc 'Choose using glob patterns. Adding capabilities to handle [] and ?, and {}.'
+    config_param :glob_policy, :enum, list: [:backward_compatible, :extended, :always], default: :backward_compatible
     desc 'The tag of the event.'
     config_param :tag, :string
     desc 'The paths to exclude the files from watcher list.'
@@ -141,6 +143,14 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "either of enable_watch_timer or enable_stat_watcher must be true"
       end
 
+      if @glob_policy == :always && @path_delimiter == ','
+        raise Fluent::ConfigError, "cannot use glob_policy as always with the default path_delimitor: `,\""
+      end
+
+      if @glob_policy == :extended && /\{.*,.*\}/.match(@path) && extended_glob_pattern(@path)
+        raise Fluent::ConfigError, "cannot include curly braces with glob patterns in `#{@path}\". Use glob_policy always instead."
+      end
+
       if RESERVED_CHARS.include?(@path_delimiter)
         rc = RESERVED_CHARS.join(', ')
         raise Fluent::ConfigError, "#{rc} are reserved words: #{@path_delimiter}"
@@ -288,6 +298,28 @@ module Fluent::Plugin
         @capability.have_capability?(:effective, :dac_override)
     end
 
+    def extended_glob_pattern(path)
+      path.include?('*') || path.include?('?') || /\[.*\]/.match(path)
+    end
+
+    # Curly braces is not supported with default path_delimiter
+    # because the default delimiter of path is ",".
+    # This should be collided for wildcard pattern for curly braces and
+    # be handled as an error on #configure.
+    def use_glob?(path)
+      if @glob_policy == :always
+        # For future extensions, we decided to use `always' term to handle
+        # regular expressions as much as possible.
+        # This is because not using `true' as a returning value
+        # when choosing :always here.
+        extended_glob_pattern(path) || /\{.*,.*\}/.match(path)
+      elsif @glob_policy == :extended
+        extended_glob_pattern(path)
+      elsif @glob_policy == :backward_compatible
+        path.include?('*')
+      end
+    end
+
     def expand_paths
       date = Fluent::EventTime.now
       paths = []
@@ -297,7 +329,7 @@ module Fluent::Plugin
                else
                  date.to_time.strftime(path)
                end
-        if path.include?('*')
+        if use_glob?(path)
           paths += Dir.glob(path).select { |p|
             begin
               is_file = !File.directory?(p)
@@ -332,7 +364,7 @@ module Fluent::Plugin
                else
                  date.to_time.strftime(path)
                end
-        path.include?('*') ? Dir.glob(path) : path
+        use_glob?(path) ? Dir.glob(path) : path
       }.flatten.uniq
       # filter out non existing files, so in case pattern is without '*' we don't do unnecessary work
       hash = {}

data/lib/fluent/plugin/out_http.rb

@@ -37,6 +37,8 @@ module Fluent::Plugin
 
     class RetryableResponse < StandardError; end
 
+    ConnectionCache = Struct.new(:uri, :conn)
+
     helpers :formatter
 
     desc 'The endpoint for HTTP request, e.g. http://example.com/api'
@@ -60,6 +62,8 @@ module Fluent::Plugin
     config_param :read_timeout, :integer, default: nil
     desc 'The TLS timeout in seconds'
     config_param :ssl_timeout, :integer, default: nil
+    desc 'Try to reuse connections'
+    config_param :reuse_connections, :bool, default: false
 
     desc 'The CA certificate path for TLS'
     config_param :tls_ca_cert_path, :string, default: nil
@@ -87,11 +91,29 @@ module Fluent::Plugin
 
     config_section :auth, required: false, multi: false do
       desc 'The method for HTTP authentication'
-      config_param :method, :enum, list: [:basic], default: :basic
+      config_param :method, :enum, list: [:basic, :aws_sigv4], default: :basic
       desc 'The username for basic authentication'
       config_param :username, :string, default: nil
       desc 'The password for basic authentication'
       config_param :password, :string, default: nil, secret: true
+      desc 'The AWS service to authenticate against'
+      config_param :aws_service, :string, default: nil
+      desc 'The AWS region to use when authenticating'
+      config_param :aws_region, :string, default: nil
+      desc 'The AWS role ARN to assume when authenticating'
+      config_param :aws_role_arn, :string, default: nil
+    end
+
+    def connection_cache_id_thread_key
+      "#{plugin_id}_connection_cache_id"
+    end
+
+    def connection_cache_id_for_thread
+      Thread.current[connection_cache_id_thread_key]
+    end
+
+    def connection_cache_id_for_thread=(id)
+      Thread.current[connection_cache_id_thread_key] = id
     end
 
     def initialize
@@ -100,11 +122,23 @@ module Fluent::Plugin
       @uri = nil
       @proxy_uri = nil
       @formatter = nil
+
+      @connection_cache = []
+      @connection_cache_id_mutex = Mutex.new
+      @connection_cache_next_id = 0
+    end
+
+    def close
+      super
+
+      @connection_cache.each {|entry| entry.conn.finish if entry.conn&.started? }
     end
 
     def configure(conf)
       super
 
+      @connection_cache = Array.new(actual_flush_thread_count, ConnectionCache.new("", nil)) if @reuse_connections
+
       if @retryable_response_codes.nil?
         log.warn('Status code 503 is going to be removed from default `retryable_response_codes` from fluentd v2. Please add it by yourself if you wish')
         @retryable_response_codes = [503]
@@ -121,6 +155,36 @@ module Fluent::Plugin
         end
         define_singleton_method(:format, method(:format_json_array))
       end
+
+      if @auth and @auth.method == :aws_sigv4
+        begin
+          require 'aws-sigv4'
+          require 'aws-sdk-core'
+        rescue LoadError
+          raise Fluent::ConfigError, "The aws-sdk-core and aws-sigv4 gems are required for aws_sigv4 auth. Run: gem install aws-sdk-core -v '~> 3.191'"
+        end
+
+        raise Fluent::ConfigError, "aws_service is required for aws_sigv4 auth" unless @auth.aws_service != nil
+        raise Fluent::ConfigError, "aws_region is required for aws_sigv4 auth" unless @auth.aws_region != nil
+
+        if @auth.aws_role_arn == nil
+          aws_credentials = Aws::CredentialProviderChain.new.resolve
+        else
+          aws_credentials = Aws::AssumeRoleCredentials.new(
+            client: Aws::STS::Client.new(
+              region: @auth.aws_region
+            ),
+            role_arn: @auth.aws_role_arn,
+            role_session_name: "fluentd"
+          )
+        end
+
+        @aws_signer = Aws::Sigv4::Signer.new(
+          service: @auth.aws_service,
+          region: @auth.aws_region,
+          credentials_provider: aws_credentials
+        )
+      end
     end
 
     def multi_workers_ready?
@@ -215,7 +279,7 @@ module Fluent::Plugin
       URI.parse(endpoint)
     end
 
-    def set_headers(req, chunk)
+    def set_headers(req, uri, chunk)
       if @headers
         @headers.each do |k, v|
           req[k] = v
@@ -229,6 +293,28 @@ module Fluent::Plugin
       req['Content-Type'] = @content_type
     end
 
+    def set_auth(req, uri)
+      return unless @auth
+
+      if @auth.method == :basic
+        req.basic_auth(@auth.username, @auth.password)
+      elsif @auth.method == :aws_sigv4
+        signature = @aws_signer.sign_request(
+          http_method: req.method,
+          url: uri.request_uri,
+          headers: {
+            'Content-Type' => @content_type,
+            'Host' => uri.host
+          },
+          body: req.body
+        )
+        req.add_field('x-amz-date', signature.headers['x-amz-date'])
+        req.add_field('x-amz-security-token', signature.headers['x-amz-security-token'])
+        req.add_field('x-amz-content-sha256', signature.headers['x-amz-content-sha256'])
+        req.add_field('authorization', signature.headers['authorization'])
+      end
+    end
+
     def create_request(chunk, uri)
       req = case @http_method
             when :post
@@ -236,23 +322,49 @@ module Fluent::Plugin
             when :put
               Net::HTTP::Put.new(uri.request_uri)
             end
-      if @auth
-        req.basic_auth(@auth.username, @auth.password)
-      end
-      set_headers(req, chunk)
+      set_headers(req, uri, chunk)
       req.body = @json_array ? "[#{chunk.read.chop}]" : chunk.read
+
+      # At least one authentication method requires the body and other headers, so the order of this call matters
+      set_auth(req, uri)
       req
     end
 
+    def make_request_cached(uri, req)
+      id = self.connection_cache_id_for_thread
+      if id.nil?
+        @connection_cache_id_mutex.synchronize {
+          id = @connection_cache_next_id
+          @connection_cache_next_id += 1
+        }
+        self.connection_cache_id_for_thread = id
+      end
+      uri_str = uri.to_s
+      if @connection_cache[id].uri != uri_str
+        @connection_cache[id].conn.finish if @connection_cache[id].conn&.started?
+        http =  if @proxy_uri
+                  Net::HTTP.start(uri.host, uri.port, @proxy_uri.host, @proxy_uri.port, @proxy_uri.user, @proxy_uri.password, @http_opt)
+                else
+                  Net::HTTP.start(uri.host, uri.port, @http_opt)
+                end
+        @connection_cache[id] = ConnectionCache.new(uri_str, http)
+      end
+      @connection_cache[id].conn.request(req)
+    end
+
+    def make_request(uri, req, &block)
+      if @proxy_uri
+        Net::HTTP.start(uri.host, uri.port, @proxy_uri.host, @proxy_uri.port, @proxy_uri.user, @proxy_uri.password, @http_opt, &block)
+      else
+        Net::HTTP.start(uri.host, uri.port, @http_opt, &block)
+      end
+    end
+
     def send_request(uri, req)
-      res = if @proxy_uri
-              Net::HTTP.start(uri.host, uri.port, @proxy_uri.host, @proxy_uri.port, @proxy_uri.user, @proxy_uri.password, @http_opt) { |http|
-                http.request(req)
-              }
+      res = if @reuse_connections
+              make_request_cached(uri, req)
             else
-              Net::HTTP.start(uri.host, uri.port, @http_opt) { |http|
-                http.request(req)
-              }
+              make_request(uri, req) { |http| http.request(req) }
             end
 
       if res.is_a?(Net::HTTPSuccess)

data/lib/fluent/plugin/owned_by_mixin.rb

@@ -21,7 +21,6 @@ module Fluent
         @_owner = plugin
 
         @_plugin_id = plugin.plugin_id
-        @_plugin_id_configured = plugin.plugin_id_configured?
 
         @log = plugin.log
       end

data/lib/fluent/plugin/parser_json.rb

@@ -70,16 +70,33 @@ module Fluent
       end
 
       def parse(text)
-        record = @load_proc.call(text)
-        time = parse_time(record)
-        if @execute_convert_values
-          time, record = convert_values(time, record)
+        parsed_json = @load_proc.call(text)
+
+        if parsed_json.is_a?(Hash)
+          time, record = parse_one_record(parsed_json)
+          yield time, record
+        elsif parsed_json.is_a?(Array)
+          parsed_json.each do |record|
+            unless record.is_a?(Hash)
+              yield nil, nil
+              next
+            end
+            time, parsed_record = parse_one_record(record)
+            yield time, parsed_record
+          end
+        else
+          yield nil, nil
         end
-        yield time, record
+
       rescue @error_class, EncodingError # EncodingError is for oj 3.x or later
         yield nil, nil
       end
 
+      def parse_one_record(record)
+        time = parse_time(record)
+        convert_values(time, record)
+      end
+
       def parser_type
         :text
       end

data/lib/fluent/plugin/parser_msgpack.rb

@@ -31,9 +31,9 @@ module Fluent
         :binary
       end
 
-      def parse(data)
+      def parse(data, &block)
         @unpacker.feed_each(data) do |obj|
-          yield convert_values(parse_time(obj), obj)
+          parse_unpacked_data(obj, &block)
         end
       end
       alias parse_partial_data parse
@@ -41,8 +41,29 @@ module Fluent
       def parse_io(io, &block)
         u = Fluent::MessagePackFactory.engine_factory.unpacker(io)
         u.each do |obj|
-          time, record = convert_values(parse_time(obj), obj)
+          parse_unpacked_data(obj, &block)
+        end
+      end
+
+      def parse_unpacked_data(data)
+        if data.is_a?(Hash)
+          time, record = convert_values(parse_time(data), data)
           yield time, record
+          return
+        end
+
+        unless data.is_a?(Array)
+          yield nil, nil
+          return
+        end
+
+        data.each do |record|
+          unless record.is_a?(Hash)
+            yield nil, nil
+            next
+          end
+          time, converted_record = convert_values(parse_time(record), record)
+          yield time, converted_record
         end
       end
     end

data/lib/fluent/plugin_helper/metrics.rb

@@ -65,9 +65,9 @@ module Fluent
         metrics.configure(config)
         # For multi workers environment, cmetrics should be distinguish with static labels.
         if Fluent::Engine.system_config.workers > 1
-          labels.merge!(worker_id: fluentd_worker_id.to_s)
+          labels[:worker_id] = fluentd_worker_id.to_s
         end
-        labels.merge!(plugin: @plugin_type_or_id)
+        labels[:plugin] = @plugin_type_or_id
         metrics.create(namespace: namespace, subsystem: subsystem, name: name, help_text: help_text, labels: labels)
 
         @_metrics["#{@plugin_type_or_id}_#{namespace}_#{subsystem}_#{name}"] = metrics

data/lib/fluent/registry.rb

@@ -60,13 +60,13 @@ module Fluent
       # search from additional plugin directories
       if @dir_search_prefix
         path = "#{@dir_search_prefix}#{type}"
-        files = @paths.map { |lp|
+        files = @paths.filter_map { |lp|
           lpath = File.expand_path(File.join(lp, "#{path}.rb"))
           File.exist?(lpath) ? lpath : nil
-        }.compact
+        }
         unless files.empty?
           # prefer newer version
-          require files.sort.last
+          require files.max
           return
         end
       end
@@ -74,17 +74,17 @@ module Fluent
       path = "#{@search_prefix}#{type}"
 
       # prefer LOAD_PATH than gems
-      files = $LOAD_PATH.map { |lp|
+      files = $LOAD_PATH.filter_map { |lp|
         if lp == FLUENT_LIB_PATH
           nil
         else
           lpath = File.expand_path(File.join(lp, "#{path}.rb"))
           File.exist?(lpath) ? lpath : nil
         end
-      }.compact
+      }
       unless files.empty?
         # prefer newer version
-        require files.sort.last
+        require files.max
         return
       end
 

data/lib/fluent/test/output_test.rb

@@ -139,7 +139,7 @@ module Fluent
             assert_equal(@expected_buffer, buffer)
           end
 
-          lines.keys.each do |meta|
+          lines.each_key do |meta|
             chunk = @instance.buffer.generate_chunk(meta).staged!
             chunk.append(lines[meta])
             begin

data/lib/fluent/unique_id.rb

@@ -23,7 +23,7 @@ module Fluent
     end
 
     def self.hex(unique_id)
-      unique_id.unpack('H*').first
+      unique_id.unpack1('H*')
     end
 
     module Mixin

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.16.4'
+  VERSION = '1.17.0'
 
 end

data/test/log/test_console_adapter.rb

@@ -72,11 +72,18 @@ class ConsoleAdapterTest < Test::Unit::TestCase
        fatal: :fatal)
   def test_options(level)
     @console_logger.send(level, "subject", kwarg1: "opt1", kwarg2: "opt2")
+    lines = @logdev.logs[0].split("\n")
+    args = JSON.load(lines[1..].collect { |str| str.sub(/\s+\|/, "") }.join("\n"));
     assert_equal([
-                   "#{@timestamp_str} [#{level}]:   0.0s: subject\n" +
-                   "      | {\"kwarg1\":\"opt1\",\"kwarg2\":\"opt2\"}\n"
+                   1,
+                   "#{@timestamp_str} [#{level}]:   0.0s: subject",
+                   { "kwarg1" => "opt1", "kwarg2" => "opt2" }
                  ],
-                 @logdev.logs)
+                 [
+                   @logdev.logs.size,
+                   lines[0],
+                   args
+                 ])
   end
 
   data(debug: :debug,

data/test/plugin/test_buffer.rb

@@ -901,6 +901,65 @@ class BufferTest < Test::Unit::TestCase
 
       assert_equal 2, purge_count
     end
+
+    # https://github.com/fluent/fluentd/issues/4446
+    test "#write_step_by_step keeps chunks kept in locked in entire #write process" do
+      assert_equal 8 * 1024 * 1024, @p.chunk_limit_size
+      assert_equal 0.95, @p.chunk_full_threshold
+
+      mon_enter_counts_by_chunk = {}
+      mon_exit_counts_by_chunk = {}
+
+      stub.proxy(@p).generate_chunk(anything) do |chunk|
+        stub(chunk).mon_enter do
+          enter_count = 1 + mon_enter_counts_by_chunk.fetch(chunk, 0)
+          exit_count = mon_exit_counts_by_chunk.fetch(chunk, 0)
+          mon_enter_counts_by_chunk[chunk] = enter_count
+
+          # Assert that chunk is passed to &block of write_step_by_step before exiting the lock.
+          # (i.e. The lock count must be 2 greater than the exit count).
+          # Since ShouldRetry occurs once, the staged chunk takes the lock 3 times when calling the block.
+          if chunk.staged?
+            lock_in_block = enter_count == 3
+            assert_equal(enter_count - 2, exit_count) if lock_in_block
+          else
+            lock_in_block = enter_count == 2
+            assert_equal(enter_count - 2, exit_count) if lock_in_block
+          end
+        end
+        stub(chunk).mon_exit do
+          exit_count = 1 + mon_exit_counts_by_chunk.fetch(chunk, 0)
+          mon_exit_counts_by_chunk[chunk] = exit_count
+        end
+        chunk
+      end
+
+      m = @p.metadata(timekey: Time.parse('2016-04-11 16:40:00 +0000').to_i)
+      small_row = "x" * 1024 * 400
+      big_row = "x" * 1024 * 1024 * 8 # just `chunk_size_limit`, it does't cause BufferOverFlowError.
+
+      # Write 42 events in 1 event stream, last one is for triggering `ShouldRetry`
+      @p.write({m => [small_row] * 40 + [big_row] + ["x"]})
+
+      # Above event strem will be splitted twice by `Buffer#write_step_by_step`
+      #
+      # 1. `write_once`: 42 [events] * 1 [stream]
+      # 2. `write_step_by_step`: 4 [events]* 10 [streams] + 2 [events] * 1 [stream]
+      # 3. `write_step_by_step` (by `ShouldRetry`): 1 [event] * 42 [streams]
+      # 
+      # Example of staged chunk lock behavior:
+      # 
+      # 1. mon_enter in write_step_by_step
+      # 2. ShouldRetry occurs
+      # 3. mon_exit in write_step_by_step
+      # 4. mon_enter again in write_step_by_step (retry)
+      # 5. passed to &block of write_step_by_step
+      # 6. mon_enter in the block (write)
+      # 7. mon_exit in write_step_by_step
+      # 8. mon_exit in write
+
+      assert_equal(mon_enter_counts_by_chunk.values, mon_exit_counts_by_chunk.values)
+    end
   end
 
   sub_test_case 'standard format with configuration for test with lower chunk limit size' do