fluentd 1.16.6-x86-mingw32 → 1.17.0-x86-mingw32

data/lib/fluent/plugin/in_tail.rb

@@ -65,6 +65,8 @@ module Fluent::Plugin
     config_param :path, :string
     desc 'path delimiter used for spliting path config'
     config_param :path_delimiter, :string, default: ','
+    desc 'Choose using glob patterns. Adding capabilities to handle [] and ?, and {}.'
+    config_param :glob_policy, :enum, list: [:backward_compatible, :extended, :always], default: :backward_compatible
     desc 'The tag of the event.'
     config_param :tag, :string
     desc 'The paths to exclude the files from watcher list.'
@@ -141,6 +143,14 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "either of enable_watch_timer or enable_stat_watcher must be true"
       end
 
+      if @glob_policy == :always && @path_delimiter == ','
+        raise Fluent::ConfigError, "cannot use glob_policy as always with the default path_delimitor: `,\""
+      end
+
+      if @glob_policy == :extended && /\{.*,.*\}/.match(@path) && extended_glob_pattern(@path)
+        raise Fluent::ConfigError, "cannot include curly braces with glob patterns in `#{@path}\". Use glob_policy always instead."
+      end
+
       if RESERVED_CHARS.include?(@path_delimiter)
         rc = RESERVED_CHARS.join(', ')
         raise Fluent::ConfigError, "#{rc} are reserved words: #{@path_delimiter}"
@@ -288,6 +298,28 @@ module Fluent::Plugin
         @capability.have_capability?(:effective, :dac_override)
     end
 
+    def extended_glob_pattern(path)
+      path.include?('*') || path.include?('?') || /\[.*\]/.match(path)
+    end
+
+    # Curly braces is not supported with default path_delimiter
+    # because the default delimiter of path is ",".
+    # This should be collided for wildcard pattern for curly braces and
+    # be handled as an error on #configure.
+    def use_glob?(path)
+      if @glob_policy == :always
+        # For future extensions, we decided to use `always' term to handle
+        # regular expressions as much as possible.
+        # This is because not using `true' as a returning value
+        # when choosing :always here.
+        extended_glob_pattern(path) || /\{.*,.*\}/.match(path)
+      elsif @glob_policy == :extended
+        extended_glob_pattern(path)
+      elsif @glob_policy == :backward_compatible
+        path.include?('*')
+      end
+    end
+
     def expand_paths
       date = Fluent::EventTime.now
       paths = []
@@ -297,7 +329,7 @@ module Fluent::Plugin
                else
                  date.to_time.strftime(path)
                end
-        if path.include?('*')
+        if use_glob?(path)
           paths += Dir.glob(path).select { |p|
             begin
               is_file = !File.directory?(p)
@@ -332,7 +364,7 @@ module Fluent::Plugin
                else
                  date.to_time.strftime(path)
                end
-        path.include?('*') ? Dir.glob(path) : path
+        use_glob?(path) ? Dir.glob(path) : path
       }.flatten.uniq
       # filter out non existing files, so in case pattern is without '*' we don't do unnecessary work
       hash = {}

data/lib/fluent/plugin/out_file.rb

@@ -172,14 +172,6 @@ module Fluent::Plugin
           log.warn "symlink_path is unavailable on Windows platform. disabled."
           @symlink_path = nil
         else
-          placeholder_validators(:symlink_path, @symlink_path).reject{ |v| v.type == :time }.each do |v|
-            begin
-              v.validate!
-            rescue Fluent::ConfigError => e
-              log.warn "#{e}. This means multiple chunks are competing for a single symlink_path, so some logs may not be taken from the symlink."
-            end
-          end
-
           @buffer.extend SymlinkBufferMixin
           @buffer.symlink_path = @symlink_path
           @buffer.output_plugin_for_symlink = self

data/lib/fluent/plugin/out_http.rb

@@ -37,6 +37,8 @@ module Fluent::Plugin
 
     class RetryableResponse < StandardError; end
 
+    ConnectionCache = Struct.new(:uri, :conn)
+
     helpers :formatter
 
     desc 'The endpoint for HTTP request, e.g. http://example.com/api'
@@ -60,6 +62,8 @@ module Fluent::Plugin
     config_param :read_timeout, :integer, default: nil
     desc 'The TLS timeout in seconds'
     config_param :ssl_timeout, :integer, default: nil
+    desc 'Try to reuse connections'
+    config_param :reuse_connections, :bool, default: false
 
     desc 'The CA certificate path for TLS'
     config_param :tls_ca_cert_path, :string, default: nil
@@ -87,11 +91,29 @@ module Fluent::Plugin
 
     config_section :auth, required: false, multi: false do
       desc 'The method for HTTP authentication'
-      config_param :method, :enum, list: [:basic], default: :basic
+      config_param :method, :enum, list: [:basic, :aws_sigv4], default: :basic
       desc 'The username for basic authentication'
       config_param :username, :string, default: nil
       desc 'The password for basic authentication'
       config_param :password, :string, default: nil, secret: true
+      desc 'The AWS service to authenticate against'
+      config_param :aws_service, :string, default: nil
+      desc 'The AWS region to use when authenticating'
+      config_param :aws_region, :string, default: nil
+      desc 'The AWS role ARN to assume when authenticating'
+      config_param :aws_role_arn, :string, default: nil
+    end
+
+    def connection_cache_id_thread_key
+      "#{plugin_id}_connection_cache_id"
+    end
+
+    def connection_cache_id_for_thread
+      Thread.current[connection_cache_id_thread_key]
+    end
+
+    def connection_cache_id_for_thread=(id)
+      Thread.current[connection_cache_id_thread_key] = id
     end
 
     def initialize
@@ -100,11 +122,23 @@ module Fluent::Plugin
       @uri = nil
       @proxy_uri = nil
       @formatter = nil
+
+      @connection_cache = []
+      @connection_cache_id_mutex = Mutex.new
+      @connection_cache_next_id = 0
+    end
+
+    def close
+      super
+
+      @connection_cache.each {|entry| entry.conn.finish if entry.conn&.started? }
     end
 
     def configure(conf)
       super
 
+      @connection_cache = Array.new(actual_flush_thread_count, ConnectionCache.new("", nil)) if @reuse_connections
+
       if @retryable_response_codes.nil?
         log.warn('Status code 503 is going to be removed from default `retryable_response_codes` from fluentd v2. Please add it by yourself if you wish')
         @retryable_response_codes = [503]
@@ -121,6 +155,36 @@ module Fluent::Plugin
         end
         define_singleton_method(:format, method(:format_json_array))
       end
+
+      if @auth and @auth.method == :aws_sigv4
+        begin
+          require 'aws-sigv4'
+          require 'aws-sdk-core'
+        rescue LoadError
+          raise Fluent::ConfigError, "The aws-sdk-core and aws-sigv4 gems are required for aws_sigv4 auth. Run: gem install aws-sdk-core -v '~> 3.191'"
+        end
+
+        raise Fluent::ConfigError, "aws_service is required for aws_sigv4 auth" unless @auth.aws_service != nil
+        raise Fluent::ConfigError, "aws_region is required for aws_sigv4 auth" unless @auth.aws_region != nil
+
+        if @auth.aws_role_arn == nil
+          aws_credentials = Aws::CredentialProviderChain.new.resolve
+        else
+          aws_credentials = Aws::AssumeRoleCredentials.new(
+            client: Aws::STS::Client.new(
+              region: @auth.aws_region
+            ),
+            role_arn: @auth.aws_role_arn,
+            role_session_name: "fluentd"
+          )
+        end
+
+        @aws_signer = Aws::Sigv4::Signer.new(
+          service: @auth.aws_service,
+          region: @auth.aws_region,
+          credentials_provider: aws_credentials
+        )
+      end
     end
 
     def multi_workers_ready?
@@ -215,7 +279,7 @@ module Fluent::Plugin
       URI.parse(endpoint)
     end
 
-    def set_headers(req, chunk)
+    def set_headers(req, uri, chunk)
       if @headers
         @headers.each do |k, v|
           req[k] = v
@@ -229,6 +293,28 @@ module Fluent::Plugin
       req['Content-Type'] = @content_type
     end
 
+    def set_auth(req, uri)
+      return unless @auth
+
+      if @auth.method == :basic
+        req.basic_auth(@auth.username, @auth.password)
+      elsif @auth.method == :aws_sigv4
+        signature = @aws_signer.sign_request(
+          http_method: req.method,
+          url: uri.request_uri,
+          headers: {
+            'Content-Type' => @content_type,
+            'Host' => uri.host
+          },
+          body: req.body
+        )
+        req.add_field('x-amz-date', signature.headers['x-amz-date'])
+        req.add_field('x-amz-security-token', signature.headers['x-amz-security-token'])
+        req.add_field('x-amz-content-sha256', signature.headers['x-amz-content-sha256'])
+        req.add_field('authorization', signature.headers['authorization'])
+      end
+    end
+
     def create_request(chunk, uri)
       req = case @http_method
             when :post
@@ -236,23 +322,49 @@ module Fluent::Plugin
             when :put
               Net::HTTP::Put.new(uri.request_uri)
             end
-      if @auth
-        req.basic_auth(@auth.username, @auth.password)
-      end
-      set_headers(req, chunk)
+      set_headers(req, uri, chunk)
       req.body = @json_array ? "[#{chunk.read.chop}]" : chunk.read
+
+      # At least one authentication method requires the body and other headers, so the order of this call matters
+      set_auth(req, uri)
       req
     end
 
+    def make_request_cached(uri, req)
+      id = self.connection_cache_id_for_thread
+      if id.nil?
+        @connection_cache_id_mutex.synchronize {
+          id = @connection_cache_next_id
+          @connection_cache_next_id += 1
+        }
+        self.connection_cache_id_for_thread = id
+      end
+      uri_str = uri.to_s
+      if @connection_cache[id].uri != uri_str
+        @connection_cache[id].conn.finish if @connection_cache[id].conn&.started?
+        http =  if @proxy_uri
+                  Net::HTTP.start(uri.host, uri.port, @proxy_uri.host, @proxy_uri.port, @proxy_uri.user, @proxy_uri.password, @http_opt)
+                else
+                  Net::HTTP.start(uri.host, uri.port, @http_opt)
+                end
+        @connection_cache[id] = ConnectionCache.new(uri_str, http)
+      end
+      @connection_cache[id].conn.request(req)
+    end
+
+    def make_request(uri, req, &block)
+      if @proxy_uri
+        Net::HTTP.start(uri.host, uri.port, @proxy_uri.host, @proxy_uri.port, @proxy_uri.user, @proxy_uri.password, @http_opt, &block)
+      else
+        Net::HTTP.start(uri.host, uri.port, @http_opt, &block)
+      end
+    end
+
     def send_request(uri, req)
-      res = if @proxy_uri
-              Net::HTTP.start(uri.host, uri.port, @proxy_uri.host, @proxy_uri.port, @proxy_uri.user, @proxy_uri.password, @http_opt) { |http|
-                http.request(req)
-              }
+      res = if @reuse_connections
+              make_request_cached(uri, req)
             else
-              Net::HTTP.start(uri.host, uri.port, @http_opt) { |http|
-                http.request(req)
-              }
+              make_request(uri, req) { |http| http.request(req) }
             end
 
       if res.is_a?(Net::HTTPSuccess)

data/lib/fluent/plugin/owned_by_mixin.rb

@@ -21,7 +21,6 @@ module Fluent
         @_owner = plugin
 
         @_plugin_id = plugin.plugin_id
-        @_plugin_id_configured = plugin.plugin_id_configured?
 
         @log = plugin.log
       end

data/lib/fluent/plugin/parser_json.rb

@@ -50,28 +50,53 @@ module Fluent
       def configure_json_parser(name)
         case name
         when :oj
-          return [Oj.method(:load), Oj::ParseError] if Fluent::OjOptions.available?
-
-          log&.info "Oj is not installed, and failing back to Yajl for json parser"
-          configure_json_parser(:yajl)
+          raise LoadError unless Fluent::OjOptions.available?
+          [Oj.method(:load), Oj::ParseError]
         when :json then [JSON.method(:load), JSON::ParserError]
         when :yajl then [Yajl.method(:load), Yajl::ParseError]
         else
           raise "BUG: unknown json parser specified: #{name}"
         end
+      rescue LoadError => ex
+        name = :yajl
+        if log
+          if /\boj\z/.match?(ex.message)
+            log.info "Oj is not installed, and failing back to Yajl for json parser"
+          else
+            log.warn ex.message
+          end
+        end
+        retry
       end
 
       def parse(text)
-        record = @load_proc.call(text)
-        time = parse_time(record)
-        if @execute_convert_values
-          time, record = convert_values(time, record)
+        parsed_json = @load_proc.call(text)
+
+        if parsed_json.is_a?(Hash)
+          time, record = parse_one_record(parsed_json)
+          yield time, record
+        elsif parsed_json.is_a?(Array)
+          parsed_json.each do |record|
+            unless record.is_a?(Hash)
+              yield nil, nil
+              next
+            end
+            time, parsed_record = parse_one_record(record)
+            yield time, parsed_record
+          end
+        else
+          yield nil, nil
         end
-        yield time, record
+
       rescue @error_class, EncodingError # EncodingError is for oj 3.x or later
         yield nil, nil
       end
 
+      def parse_one_record(record)
+        time = parse_time(record)
+        convert_values(time, record)
+      end
+
       def parser_type
         :text
       end

data/lib/fluent/plugin/parser_msgpack.rb

@@ -31,9 +31,9 @@ module Fluent
         :binary
       end
 
-      def parse(data)
+      def parse(data, &block)
         @unpacker.feed_each(data) do |obj|
-          yield convert_values(parse_time(obj), obj)
+          parse_unpacked_data(obj, &block)
         end
       end
       alias parse_partial_data parse
@@ -41,8 +41,29 @@ module Fluent
       def parse_io(io, &block)
         u = Fluent::MessagePackFactory.engine_factory.unpacker(io)
         u.each do |obj|
-          time, record = convert_values(parse_time(obj), obj)
+          parse_unpacked_data(obj, &block)
+        end
+      end
+
+      def parse_unpacked_data(data)
+        if data.is_a?(Hash)
+          time, record = convert_values(parse_time(data), data)
           yield time, record
+          return
+        end
+
+        unless data.is_a?(Array)
+          yield nil, nil
+          return
+        end
+
+        data.each do |record|
+          unless record.is_a?(Hash)
+            yield nil, nil
+            next
+          end
+          time, converted_record = convert_values(parse_time(record), record)
+          yield time, converted_record
         end
       end
     end

data/lib/fluent/plugin_helper/metrics.rb

@@ -65,9 +65,9 @@ module Fluent
         metrics.configure(config)
         # For multi workers environment, cmetrics should be distinguish with static labels.
         if Fluent::Engine.system_config.workers > 1
-          labels.merge!(worker_id: fluentd_worker_id.to_s)
+          labels[:worker_id] = fluentd_worker_id.to_s
         end
-        labels.merge!(plugin: @plugin_type_or_id)
+        labels[:plugin] = @plugin_type_or_id
         metrics.create(namespace: namespace, subsystem: subsystem, name: name, help_text: help_text, labels: labels)
 
         @_metrics["#{@plugin_type_or_id}_#{namespace}_#{subsystem}_#{name}"] = metrics

data/lib/fluent/registry.rb

@@ -60,13 +60,13 @@ module Fluent
       # search from additional plugin directories
       if @dir_search_prefix
         path = "#{@dir_search_prefix}#{type}"
-        files = @paths.map { |lp|
+        files = @paths.filter_map { |lp|
           lpath = File.expand_path(File.join(lp, "#{path}.rb"))
           File.exist?(lpath) ? lpath : nil
-        }.compact
+        }
         unless files.empty?
           # prefer newer version
-          require files.sort.last
+          require files.max
           return
         end
       end
@@ -74,17 +74,17 @@ module Fluent
       path = "#{@search_prefix}#{type}"
 
       # prefer LOAD_PATH than gems
-      files = $LOAD_PATH.map { |lp|
+      files = $LOAD_PATH.filter_map { |lp|
         if lp == FLUENT_LIB_PATH
           nil
         else
           lpath = File.expand_path(File.join(lp, "#{path}.rb"))
           File.exist?(lpath) ? lpath : nil
         end
-      }.compact
+      }
       unless files.empty?
         # prefer newer version
-        require files.sort.last
+        require files.max
         return
       end
 

data/lib/fluent/test/output_test.rb

@@ -139,7 +139,7 @@ module Fluent
             assert_equal(@expected_buffer, buffer)
           end
 
-          lines.keys.each do |meta|
+          lines.each_key do |meta|
             chunk = @instance.buffer.generate_chunk(meta).staged!
             chunk.append(lines[meta])
             begin

data/lib/fluent/unique_id.rb

@@ -23,7 +23,7 @@ module Fluent
     end
 
     def self.hex(unique_id)
-      unique_id.unpack('H*').first
+      unique_id.unpack1('H*')
     end
 
     module Mixin

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.16.6'
+  VERSION = '1.17.0'
 
 end

data/test/command/test_fluentd.rb

@@ -128,14 +128,11 @@ class TestFluentdCommand < ::Test::Unit::TestCase
 
   # ATTENTION: This stops taking logs when all `pattern_list` match or timeout,
   # so `patterns_not_match` can test only logs up to that point.
-  # You can pass a block to assert something after log matching.
   def assert_log_matches(cmdline, *pattern_list, patterns_not_match: [], timeout: 20, env: {})
     matched = false
     matched_wrongly = false
-    error_msg_match = ""
+    assert_error_msg = ""
     stdio_buf = ""
-    succeeded_block = true
-    error_msg_block = ""
     begin
       execute_command(cmdline, @tmp_dir, env) do |pid, stdout|
         begin
@@ -166,13 +163,6 @@ class TestFluentdCommand < ::Test::Unit::TestCase
               end
             end
           end
-
-          begin
-            yield if block_given?
-          rescue => e
-            succeeded_block = false
-            error_msg_block = "failed block execution after matching: #{e}"
-          end
         ensure
           if SUPERVISOR_PID_PATTERN =~ stdio_buf
             @supervisor_pid = $1.to_i
@@ -183,19 +173,19 @@ class TestFluentdCommand < ::Test::Unit::TestCase
         end
       end
     rescue Timeout::Error
-      error_msg_match = "execution timeout"
+      assert_error_msg = "execution timeout"
       # https://github.com/fluent/fluentd/issues/4095
       # On Windows, timeout without `@supervisor_pid` means that the test is invalid,
       # since the supervisor process will survive without being killed correctly.
       flunk("Invalid test: The pid of supervisor could not be taken, which is necessary on Windows.") if Fluent.windows? && @supervisor_pid.nil?
     rescue => e
-      error_msg_match = "unexpected error in launching fluentd: #{e.inspect}"
+      assert_error_msg = "unexpected error in launching fluentd: #{e.inspect}"
     else
-      error_msg_match = "log doesn't match" unless matched
+      assert_error_msg = "log doesn't match" unless matched
     end
 
     if patterns_not_match.empty?
-      error_msg_match = build_message(error_msg_match,
+      assert_error_msg = build_message(assert_error_msg,
                                        "<?>\nwas expected to include:\n<?>",
                                        stdio_buf, pattern_list)
     else
@@ -207,17 +197,16 @@ class TestFluentdCommand < ::Test::Unit::TestCase
                             lines.any?{|line| line.include?(ptn) }
                           end
         if matched_wrongly
-          error_msg_match << "\n" unless error_msg_match.empty?
-          error_msg_match << "pattern exists in logs wrongly: #{ptn}"
+          assert_error_msg << "\n" unless assert_error_msg.empty?
+          assert_error_msg << "pattern exists in logs wrongly: #{ptn}"
         end
       end
-      error_msg_match = build_message(error_msg_match,
+      assert_error_msg = build_message(assert_error_msg,
                                        "<?>\nwas expected to include:\n<?>\nand not include:\n<?>",
                                        stdio_buf, pattern_list, patterns_not_match)
     end
 
-    assert matched && !matched_wrongly, error_msg_match
-    assert succeeded_block, error_msg_block if block_given?
+    assert matched && !matched_wrongly, assert_error_msg
   end
 
   def assert_fluentd_fails_to_start(cmdline, *pattern_list, timeout: 20)
@@ -1299,40 +1288,4 @@ CONF
                          "[debug]")
     end
   end
-
-  sub_test_case "plugin option" do
-    test "should be the default value when not specifying" do
-      conf_path = create_conf_file('test.conf', <<~CONF)
-        <source>
-          @type monitor_agent
-        </source>
-      CONF
-      assert File.exist?(conf_path)
-      cmdline = create_cmdline(conf_path)
-
-      assert_log_matches(cmdline, "fluentd worker is now running") do
-        response = Net::HTTP.get(URI.parse("http://localhost:24220/api/config.json"))
-        actual_conf = JSON.parse(response)
-        assert_equal Fluent::Supervisor.default_options[:plugin_dirs], actual_conf["plugin_dirs"]
-      end
-    end
-
-    data(short: "-p")
-    data(long: "--plugin")
-    test "can be added by specifying the option" do |option_name|
-      conf_path = create_conf_file('test.conf', <<~CONF)
-        <source>
-          @type monitor_agent
-        </source>
-      CONF
-      assert File.exist?(conf_path)
-      cmdline = create_cmdline(conf_path, option_name, @tmp_dir, option_name, @tmp_dir)
-
-      assert_log_matches(cmdline, "fluentd worker is now running") do
-        response = Net::HTTP.get(URI.parse("http://localhost:24220/api/config.json"))
-        actual_conf = JSON.parse(response)
-        assert_equal Fluent::Supervisor.default_options[:plugin_dirs] + [@tmp_dir, @tmp_dir], actual_conf["plugin_dirs"]
-      end
-    end
-  end
 end

data/test/log/test_console_adapter.rb

@@ -72,11 +72,18 @@ class ConsoleAdapterTest < Test::Unit::TestCase
        fatal: :fatal)
   def test_options(level)
     @console_logger.send(level, "subject", kwarg1: "opt1", kwarg2: "opt2")
+    lines = @logdev.logs[0].split("\n")
+    args = JSON.load(lines[1..].collect { |str| str.sub(/\s+\|/, "") }.join("\n"));
     assert_equal([
-                   "#{@timestamp_str} [#{level}]:   0.0s: subject\n" +
-                   "      | {\"kwarg1\":\"opt1\",\"kwarg2\":\"opt2\"}\n"
+                   1,
+                   "#{@timestamp_str} [#{level}]:   0.0s: subject",
+                   { "kwarg1" => "opt1", "kwarg2" => "opt2" }
                  ],
-                 @logdev.logs)
+                 [
+                   @logdev.logs.size,
+                   lines[0],
+                   args
+                 ])
   end
 
   data(debug: :debug,

data/test/plugin/in_tail/test_io_handler.rb

@@ -5,19 +5,20 @@ require 'fluent/plugin/metrics_local'
 require 'tempfile'
 
 class IntailIOHandlerTest < Test::Unit::TestCase
-  def setup
-    Tempfile.create('intail_io_handler') do |file|
-      file.binmode
-      @file = file
-      opened_file_metrics = Fluent::Plugin::LocalMetrics.new
-      opened_file_metrics.configure(config_element('metrics', '', {}))
-      closed_file_metrics = Fluent::Plugin::LocalMetrics.new
-      closed_file_metrics.configure(config_element('metrics', '', {}))
-      rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
-      rotated_file_metrics.configure(config_element('metrics', '', {}))
-      @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
-      yield
-    end
+  setup do
+    @file = Tempfile.new('intail_io_handler').binmode
+    opened_file_metrics = Fluent::Plugin::LocalMetrics.new
+    opened_file_metrics.configure(config_element('metrics', '', {}))
+    closed_file_metrics = Fluent::Plugin::LocalMetrics.new
+    closed_file_metrics.configure(config_element('metrics', '', {}))
+    rotated_file_metrics = Fluent::Plugin::LocalMetrics.new
+    rotated_file_metrics.configure(config_element('metrics', '', {}))
+    @metrics = Fluent::Plugin::TailInput::MetricsInfo.new(opened_file_metrics, closed_file_metrics, rotated_file_metrics)
+  end
+
+  teardown do
+    @file.close rescue nil
+    @file.unlink rescue nil
   end
 
   def create_target_info