fluentd 1.16.7-x64-mingw32 → 1.17.0-x64-mingw32

data/lib/fluent/plugin/in_tail.rb

@@ -65,6 +65,8 @@ module Fluent::Plugin
     config_param :path, :string
     desc 'path delimiter used for spliting path config'
     config_param :path_delimiter, :string, default: ','
+    desc 'Choose using glob patterns. Adding capabilities to handle [] and ?, and {}.'
+    config_param :glob_policy, :enum, list: [:backward_compatible, :extended, :always], default: :backward_compatible
     desc 'The tag of the event.'
     config_param :tag, :string
     desc 'The paths to exclude the files from watcher list.'
@@ -141,6 +143,14 @@ module Fluent::Plugin
         raise Fluent::ConfigError, "either of enable_watch_timer or enable_stat_watcher must be true"
       end
 
+      if @glob_policy == :always && @path_delimiter == ','
+        raise Fluent::ConfigError, "cannot use glob_policy as always with the default path_delimitor: `,\""
+      end
+
+      if @glob_policy == :extended && /\{.*,.*\}/.match(@path) && extended_glob_pattern(@path)
+        raise Fluent::ConfigError, "cannot include curly braces with glob patterns in `#{@path}\". Use glob_policy always instead."
+      end
+
       if RESERVED_CHARS.include?(@path_delimiter)
         rc = RESERVED_CHARS.join(', ')
         raise Fluent::ConfigError, "#{rc} are reserved words: #{@path_delimiter}"
@@ -288,6 +298,28 @@ module Fluent::Plugin
         @capability.have_capability?(:effective, :dac_override)
     end
 
+    def extended_glob_pattern(path)
+      path.include?('*') || path.include?('?') || /\[.*\]/.match(path)
+    end
+
+    # Curly braces is not supported with default path_delimiter
+    # because the default delimiter of path is ",".
+    # This should be collided for wildcard pattern for curly braces and
+    # be handled as an error on #configure.
+    def use_glob?(path)
+      if @glob_policy == :always
+        # For future extensions, we decided to use `always' term to handle
+        # regular expressions as much as possible.
+        # This is because not using `true' as a returning value
+        # when choosing :always here.
+        extended_glob_pattern(path) || /\{.*,.*\}/.match(path)
+      elsif @glob_policy == :extended
+        extended_glob_pattern(path)
+      elsif @glob_policy == :backward_compatible
+        path.include?('*')
+      end
+    end
+
     def expand_paths
       date = Fluent::EventTime.now
       paths = []
@@ -297,7 +329,7 @@ module Fluent::Plugin
                else
                  date.to_time.strftime(path)
                end
-        if path.include?('*')
+        if use_glob?(path)
           paths += Dir.glob(path).select { |p|
             begin
               is_file = !File.directory?(p)
@@ -332,7 +364,7 @@ module Fluent::Plugin
                else
                  date.to_time.strftime(path)
                end
-        path.include?('*') ? Dir.glob(path) : path
+        use_glob?(path) ? Dir.glob(path) : path
       }.flatten.uniq
       # filter out non existing files, so in case pattern is without '*' we don't do unnecessary work
       hash = {}

data/lib/fluent/plugin/out_file.rb

@@ -172,14 +172,6 @@ module Fluent::Plugin
           log.warn "symlink_path is unavailable on Windows platform. disabled."
           @symlink_path = nil
         else
-          placeholder_validators(:symlink_path, @symlink_path).reject{ |v| v.type == :time }.each do |v|
-            begin
-              v.validate!
-            rescue Fluent::ConfigError => e
-              log.warn "#{e}. This means multiple chunks are competing for a single symlink_path, so some logs may not be taken from the symlink."
-            end
-          end
-
           @buffer.extend SymlinkBufferMixin
           @buffer.symlink_path = @symlink_path
           @buffer.output_plugin_for_symlink = self

data/lib/fluent/plugin/out_http.rb

@@ -37,6 +37,8 @@ module Fluent::Plugin
 
     class RetryableResponse < StandardError; end
 
+    ConnectionCache = Struct.new(:uri, :conn)
+
     helpers :formatter
 
     desc 'The endpoint for HTTP request, e.g. http://example.com/api'
@@ -60,6 +62,8 @@ module Fluent::Plugin
     config_param :read_timeout, :integer, default: nil
     desc 'The TLS timeout in seconds'
     config_param :ssl_timeout, :integer, default: nil
+    desc 'Try to reuse connections'
+    config_param :reuse_connections, :bool, default: false
 
     desc 'The CA certificate path for TLS'
     config_param :tls_ca_cert_path, :string, default: nil
@@ -87,11 +91,29 @@ module Fluent::Plugin
 
     config_section :auth, required: false, multi: false do
       desc 'The method for HTTP authentication'
-      config_param :method, :enum, list: [:basic], default: :basic
+      config_param :method, :enum, list: [:basic, :aws_sigv4], default: :basic
       desc 'The username for basic authentication'
       config_param :username, :string, default: nil
       desc 'The password for basic authentication'
       config_param :password, :string, default: nil, secret: true
+      desc 'The AWS service to authenticate against'
+      config_param :aws_service, :string, default: nil
+      desc 'The AWS region to use when authenticating'
+      config_param :aws_region, :string, default: nil
+      desc 'The AWS role ARN to assume when authenticating'
+      config_param :aws_role_arn, :string, default: nil
+    end
+
+    def connection_cache_id_thread_key
+      "#{plugin_id}_connection_cache_id"
+    end
+
+    def connection_cache_id_for_thread
+      Thread.current[connection_cache_id_thread_key]
+    end
+
+    def connection_cache_id_for_thread=(id)
+      Thread.current[connection_cache_id_thread_key] = id
     end
 
     def initialize
@@ -100,11 +122,23 @@ module Fluent::Plugin
       @uri = nil
       @proxy_uri = nil
       @formatter = nil
+
+      @connection_cache = []
+      @connection_cache_id_mutex = Mutex.new
+      @connection_cache_next_id = 0
+    end
+
+    def close
+      super
+
+      @connection_cache.each {|entry| entry.conn.finish if entry.conn&.started? }
     end
 
     def configure(conf)
       super
 
+      @connection_cache = Array.new(actual_flush_thread_count, ConnectionCache.new("", nil)) if @reuse_connections
+
       if @retryable_response_codes.nil?
         log.warn('Status code 503 is going to be removed from default `retryable_response_codes` from fluentd v2. Please add it by yourself if you wish')
         @retryable_response_codes = [503]
@@ -121,6 +155,36 @@ module Fluent::Plugin
         end
         define_singleton_method(:format, method(:format_json_array))
       end
+
+      if @auth and @auth.method == :aws_sigv4
+        begin
+          require 'aws-sigv4'
+          require 'aws-sdk-core'
+        rescue LoadError
+          raise Fluent::ConfigError, "The aws-sdk-core and aws-sigv4 gems are required for aws_sigv4 auth. Run: gem install aws-sdk-core -v '~> 3.191'"
+        end
+
+        raise Fluent::ConfigError, "aws_service is required for aws_sigv4 auth" unless @auth.aws_service != nil
+        raise Fluent::ConfigError, "aws_region is required for aws_sigv4 auth" unless @auth.aws_region != nil
+
+        if @auth.aws_role_arn == nil
+          aws_credentials = Aws::CredentialProviderChain.new.resolve
+        else
+          aws_credentials = Aws::AssumeRoleCredentials.new(
+            client: Aws::STS::Client.new(
+              region: @auth.aws_region
+            ),
+            role_arn: @auth.aws_role_arn,
+            role_session_name: "fluentd"
+          )
+        end
+
+        @aws_signer = Aws::Sigv4::Signer.new(
+          service: @auth.aws_service,
+          region: @auth.aws_region,
+          credentials_provider: aws_credentials
+        )
+      end
     end
 
     def multi_workers_ready?
@@ -215,7 +279,7 @@ module Fluent::Plugin
       URI.parse(endpoint)
     end
 
-    def set_headers(req, chunk)
+    def set_headers(req, uri, chunk)
       if @headers
         @headers.each do |k, v|
           req[k] = v
@@ -229,6 +293,28 @@ module Fluent::Plugin
       req['Content-Type'] = @content_type
     end
 
+    def set_auth(req, uri)
+      return unless @auth
+
+      if @auth.method == :basic
+        req.basic_auth(@auth.username, @auth.password)
+      elsif @auth.method == :aws_sigv4
+        signature = @aws_signer.sign_request(
+          http_method: req.method,
+          url: uri.request_uri,
+          headers: {
+            'Content-Type' => @content_type,
+            'Host' => uri.host
+          },
+          body: req.body
+        )
+        req.add_field('x-amz-date', signature.headers['x-amz-date'])
+        req.add_field('x-amz-security-token', signature.headers['x-amz-security-token'])
+        req.add_field('x-amz-content-sha256', signature.headers['x-amz-content-sha256'])
+        req.add_field('authorization', signature.headers['authorization'])
+      end
+    end
+
     def create_request(chunk, uri)
       req = case @http_method
             when :post
@@ -236,23 +322,49 @@ module Fluent::Plugin
             when :put
               Net::HTTP::Put.new(uri.request_uri)
             end
-      if @auth
-        req.basic_auth(@auth.username, @auth.password)
-      end
-      set_headers(req, chunk)
+      set_headers(req, uri, chunk)
       req.body = @json_array ? "[#{chunk.read.chop}]" : chunk.read
+
+      # At least one authentication method requires the body and other headers, so the order of this call matters
+      set_auth(req, uri)
       req
     end
 
+    def make_request_cached(uri, req)
+      id = self.connection_cache_id_for_thread
+      if id.nil?
+        @connection_cache_id_mutex.synchronize {
+          id = @connection_cache_next_id
+          @connection_cache_next_id += 1
+        }
+        self.connection_cache_id_for_thread = id
+      end
+      uri_str = uri.to_s
+      if @connection_cache[id].uri != uri_str
+        @connection_cache[id].conn.finish if @connection_cache[id].conn&.started?
+        http =  if @proxy_uri
+                  Net::HTTP.start(uri.host, uri.port, @proxy_uri.host, @proxy_uri.port, @proxy_uri.user, @proxy_uri.password, @http_opt)
+                else
+                  Net::HTTP.start(uri.host, uri.port, @http_opt)
+                end
+        @connection_cache[id] = ConnectionCache.new(uri_str, http)
+      end
+      @connection_cache[id].conn.request(req)
+    end
+
+    def make_request(uri, req, &block)
+      if @proxy_uri
+        Net::HTTP.start(uri.host, uri.port, @proxy_uri.host, @proxy_uri.port, @proxy_uri.user, @proxy_uri.password, @http_opt, &block)
+      else
+        Net::HTTP.start(uri.host, uri.port, @http_opt, &block)
+      end
+    end
+
     def send_request(uri, req)
-      res = if @proxy_uri
-              Net::HTTP.start(uri.host, uri.port, @proxy_uri.host, @proxy_uri.port, @proxy_uri.user, @proxy_uri.password, @http_opt) { |http|
-                http.request(req)
-              }
+      res = if @reuse_connections
+              make_request_cached(uri, req)
             else
-              Net::HTTP.start(uri.host, uri.port, @http_opt) { |http|
-                http.request(req)
-              }
+              make_request(uri, req) { |http| http.request(req) }
             end
 
       if res.is_a?(Net::HTTPSuccess)

data/lib/fluent/plugin/owned_by_mixin.rb

@@ -21,7 +21,6 @@ module Fluent
         @_owner = plugin
 
         @_plugin_id = plugin.plugin_id
-        @_plugin_id_configured = plugin.plugin_id_configured?
 
         @log = plugin.log
       end

data/lib/fluent/plugin/parser_json.rb

@@ -50,28 +50,53 @@ module Fluent
       def configure_json_parser(name)
         case name
         when :oj
-          return [Oj.method(:load), Oj::ParseError] if Fluent::OjOptions.available?
-
-          log&.info "Oj is not installed, and failing back to Yajl for json parser"
-          configure_json_parser(:yajl)
+          raise LoadError unless Fluent::OjOptions.available?
+          [Oj.method(:load), Oj::ParseError]
         when :json then [JSON.method(:load), JSON::ParserError]
         when :yajl then [Yajl.method(:load), Yajl::ParseError]
         else
           raise "BUG: unknown json parser specified: #{name}"
         end
+      rescue LoadError => ex
+        name = :yajl
+        if log
+          if /\boj\z/.match?(ex.message)
+            log.info "Oj is not installed, and failing back to Yajl for json parser"
+          else
+            log.warn ex.message
+          end
+        end
+        retry
       end
 
       def parse(text)
-        record = @load_proc.call(text)
-        time = parse_time(record)
-        if @execute_convert_values
-          time, record = convert_values(time, record)
+        parsed_json = @load_proc.call(text)
+
+        if parsed_json.is_a?(Hash)
+          time, record = parse_one_record(parsed_json)
+          yield time, record
+        elsif parsed_json.is_a?(Array)
+          parsed_json.each do |record|
+            unless record.is_a?(Hash)
+              yield nil, nil
+              next
+            end
+            time, parsed_record = parse_one_record(record)
+            yield time, parsed_record
+          end
+        else
+          yield nil, nil
         end
-        yield time, record
+
       rescue @error_class, EncodingError # EncodingError is for oj 3.x or later
         yield nil, nil
       end
 
+      def parse_one_record(record)
+        time = parse_time(record)
+        convert_values(time, record)
+      end
+
       def parser_type
         :text
       end

data/lib/fluent/plugin/parser_msgpack.rb

@@ -31,9 +31,9 @@ module Fluent
         :binary
       end
 
-      def parse(data)
+      def parse(data, &block)
         @unpacker.feed_each(data) do |obj|
-          yield convert_values(parse_time(obj), obj)
+          parse_unpacked_data(obj, &block)
         end
       end
       alias parse_partial_data parse
@@ -41,8 +41,29 @@ module Fluent
       def parse_io(io, &block)
         u = Fluent::MessagePackFactory.engine_factory.unpacker(io)
         u.each do |obj|
-          time, record = convert_values(parse_time(obj), obj)
+          parse_unpacked_data(obj, &block)
+        end
+      end
+
+      def parse_unpacked_data(data)
+        if data.is_a?(Hash)
+          time, record = convert_values(parse_time(data), data)
           yield time, record
+          return
+        end
+
+        unless data.is_a?(Array)
+          yield nil, nil
+          return
+        end
+
+        data.each do |record|
+          unless record.is_a?(Hash)
+            yield nil, nil
+            next
+          end
+          time, converted_record = convert_values(parse_time(record), record)
+          yield time, converted_record
         end
       end
     end

data/lib/fluent/plugin_helper/metrics.rb

@@ -65,9 +65,9 @@ module Fluent
         metrics.configure(config)
         # For multi workers environment, cmetrics should be distinguish with static labels.
         if Fluent::Engine.system_config.workers > 1
-          labels.merge!(worker_id: fluentd_worker_id.to_s)
+          labels[:worker_id] = fluentd_worker_id.to_s
         end
-        labels.merge!(plugin: @plugin_type_or_id)
+        labels[:plugin] = @plugin_type_or_id
         metrics.create(namespace: namespace, subsystem: subsystem, name: name, help_text: help_text, labels: labels)
 
         @_metrics["#{@plugin_type_or_id}_#{namespace}_#{subsystem}_#{name}"] = metrics

data/lib/fluent/registry.rb

@@ -60,13 +60,13 @@ module Fluent
       # search from additional plugin directories
       if @dir_search_prefix
         path = "#{@dir_search_prefix}#{type}"
-        files = @paths.map { |lp|
+        files = @paths.filter_map { |lp|
           lpath = File.expand_path(File.join(lp, "#{path}.rb"))
           File.exist?(lpath) ? lpath : nil
-        }.compact
+        }
         unless files.empty?
           # prefer newer version
-          require files.sort.last
+          require files.max
           return
         end
       end
@@ -74,17 +74,17 @@ module Fluent
       path = "#{@search_prefix}#{type}"
 
       # prefer LOAD_PATH than gems
-      files = $LOAD_PATH.map { |lp|
+      files = $LOAD_PATH.filter_map { |lp|
         if lp == FLUENT_LIB_PATH
           nil
         else
           lpath = File.expand_path(File.join(lp, "#{path}.rb"))
           File.exist?(lpath) ? lpath : nil
         end
-      }.compact
+      }
       unless files.empty?
         # prefer newer version
-        require files.sort.last
+        require files.max
         return
       end
 

data/lib/fluent/supervisor.rb

@@ -440,7 +440,7 @@ module Fluent
         stop_immediately_at_unrecoverable_exit: true,
         root_dir: params['root_dir'],
         logger: $log,
-        log: $log&.out,
+        log: $log.out,
         log_level: params['log_level'],
         chuser: params['chuser'],
         chgroup: params['chgroup'],

data/lib/fluent/test/output_test.rb

@@ -139,7 +139,7 @@ module Fluent
             assert_equal(@expected_buffer, buffer)
           end
 
-          lines.keys.each do |meta|
+          lines.each_key do |meta|
             chunk = @instance.buffer.generate_chunk(meta).staged!
             chunk.append(lines[meta])
             begin

data/lib/fluent/unique_id.rb

@@ -23,7 +23,7 @@ module Fluent
     end
 
     def self.hex(unique_id)
-      unique_id.unpack('H*').first
+      unique_id.unpack1('H*')
     end
 
     module Mixin

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.16.7'
+  VERSION = '1.17.0'
 
 end

data/lib/fluent/winsvc.rb

@@ -63,12 +63,10 @@ begin
     end
 
     def service_stop
-      if @pid <= 0
-        set_event(@service_name)
-        return
+      set_event(@service_name)
+      if @pid > 0
+        Process.waitpid(@pid)
       end
-
-      wait_supervisor_finished
     end
 
     def service_paramchange
@@ -93,29 +91,6 @@ begin
       ev.set
       ev.close
     end
-
-    def repeat_set_event_several_times_until_success(event_name)
-      retries = 0
-      max_retries = 10
-      delay_sec = 3
-
-      begin
-        set_event(event_name)
-      rescue Errno::ENOENT
-        # This error occurs when the supervisor process has not yet created the event.
-        # If STOP is immediately executed, this state will occur.
-        # Retry `set_event' to wait for the initialization of the supervisor.
-        retries += 1
-        raise if max_retries < retries
-        sleep(delay_sec)
-        retry
-      end
-    end
-
-    def wait_supervisor_finished
-      repeat_set_event_several_times_until_success(@service_name)
-      Process.waitpid(@pid)
-    end
   end
 
   FluentdService.new(opts[:service_name]).mainloop

data/test/command/test_cat.rb

@@ -18,7 +18,7 @@ class TestFluentCat < ::Test::Unit::TestCase
     @primary = create_primary
     metadata = @primary.buffer.new_metadata
     @chunk = create_chunk(@primary, metadata, @es)
-    @port = unused_port(protocol: :all)
+    @port = unused_port
   end
 
   def teardown
@@ -87,7 +87,7 @@ class TestFluentCat < ::Test::Unit::TestCase
       d = create_driver
       d.run(expect_records: 1) do
         Open3.pipeline_w("#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} --format msgpack secondary") do |stdin|
-          stdin.write(File.read(path, File.size(path)))
+          stdin.write(File.read(path))
           stdin.close
         end
       end