fluentd 1.9.3 → 1.11.1

data/lib/fluent/plugin/sd_file.rb

@@ -85,6 +85,7 @@ module Fluent
             fetch_server_info
           rescue => e
             @log.error("sd_file: #{e}")
+            return
           end
 
         if s.nil?

data/lib/fluent/plugin/sd_srv.rb

@@ -0,0 +1,135 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+require 'resolv'
+
+require 'fluent/plugin_helper'
+require 'fluent/plugin/service_discovery'
+
+module Fluent
+  module Plugin
+    class SrvServiceDiscovery < ServiceDiscovery
+      include PluginHelper::Mixin
+
+      Plugin.register_sd('srv', self)
+
+      helpers :timer
+
+      desc 'Service without underscore in RFC2782'
+      config_param :service, :string
+      desc 'Proto without underscore in RFC2782'
+      config_param :proto, :string, default: 'tcp'
+      desc 'Name in RFC2782'
+      config_param :hostname, :string
+      desc 'hostname of DNS server to request the SRV record'
+      config_param :dns_server_host, :string, default: nil
+      desc 'interval of requesting to DNS server'
+      config_param :interval, :integer, default: 60
+      desc "resolve hostname to IP addr of SRV's Target"
+      config_param :dns_lookup, :bool, default: true
+      desc 'The shared key per server'
+      config_param :shared_key, :string, default: nil, secret: true
+      desc 'The username for authentication'
+      config_param :username, :string, default: ''
+      desc 'The password for authentication'
+      config_param :password, :string, default: '', secret: true
+
+      def initialize
+        super
+        @target = nil
+      end
+
+      def configure(conf)
+        super
+
+        @target = "_#{@service}._#{@proto}.#{@hostname}"
+        @dns_resolve =
+          if @dns_server_host.nil?
+            Resolv::DNS.new
+          elsif @dns_server_host.include?(':') # e.g. 127.0.0.1:8600
+            host, port = @dns_server_host.split(':', 2)
+            Resolv::DNS.new(nameserver_port: [[host, port.to_i]])
+          else
+            Resolv::DNS.new(nameserver: @dns_server_host)
+          end
+
+        @services = fetch_srv_record
+      end
+
+      def start(queue)
+        timer_execute(:"sd_srv_record_#{@target}", @interval) do
+          refresh_srv_records(queue)
+        end
+
+        super()
+      end
+
+      private
+
+      def refresh_srv_records(queue)
+        s = begin
+              fetch_srv_record
+            rescue => e
+              @log.error("sd_srv: #{e}")
+              return
+            end
+
+        if s.nil? || s.empty?
+          return
+        end
+
+        diff = []
+        join = s - @services
+        # Need service_in first to guarantee that server exist at least one all time.
+        join.each do |j|
+          diff << ServiceDiscovery.service_in_msg(j)
+        end
+
+        drain = @services - s
+        drain.each do |d|
+          diff << ServiceDiscovery.service_out_msg(d)
+        end
+
+        @services = s
+
+        diff.each do |a|
+          queue.push(a)
+        end
+      end
+
+      def fetch_srv_record
+        adders = @dns_resolve.getresources(@target, Resolv::DNS::Resource::IN::SRV)
+
+        services = []
+
+        adders.each do |addr|
+          host = @dns_lookup ? dns_lookup!(addr.target) : addr.target
+          services << [
+            addr.priority,
+            Service.new(:srv, host.to_s, addr.port.to_i, addr.target.to_s, addr.weight, false, @username, @password, @shared_key)
+          ]
+        end
+
+        services.sort_by(&:first).flat_map { |s| s[1] }
+      end
+
+      def dns_lookup!(host)
+        # may need to cache the result
+        @dns_resolve.getaddress(host) # get first result for now
+      end
+    end
+  end
+end

data/lib/fluent/plugin_helper/cert_option.rb

@@ -36,7 +36,7 @@ module Fluent
         end
 
         if conf.client_cert_auth
-            ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
+          ctx.verify_mode = OpenSSL::SSL::VERIFY_PEER | OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT
         end
 
         ctx.ca_file = conf.ca_path
@@ -45,6 +45,16 @@ module Fluent
         if extra && !extra.empty?
           ctx.extra_chain_cert = extra
         end
+        if conf.cert_verifier
+          sandbox = Class.new
+          ctx.verify_callback = if File.exist?(conf.cert_verifier)
+                                  verifier = File.read(conf.cert_verifier)
+                                  sandbox.instance_eval(verifier, File.basename(conf.cert_verifier))
+                                else
+                                  sandbox.instance_eval(conf.cert_verifier)
+                                end
+        end
+
         Fluent::TLS.set_version_to_context(ctx, version, conf.min_version, conf.max_version)
 
         ctx
@@ -171,9 +181,12 @@ module Fluent
 
       def cert_option_certificates_from_file(path)
         data = File.read(path)
-        pattern = Regexp.compile('-+BEGIN CERTIFICATE-+\n(?:[^-]*\n)+-+END CERTIFICATE-+\n?', Regexp::MULTILINE)
+        pattern = Regexp.compile('-+BEGIN CERTIFICATE-+\r?\n(?:[^-]*\r?\n)+-+END CERTIFICATE-+\r?\n?', Regexp::MULTILINE)
         list = []
         data.scan(pattern){|match| list << OpenSSL::X509::Certificate.new(match) }
+        if list.length == 0
+          log.warn "cert_path does not contain a valid certificate"
+        end
         list
       end
     end

data/lib/fluent/plugin_helper/child_process.rb

@@ -259,6 +259,9 @@ module Fluent
 
         if !mode.include?(:stderr) && !mode.include?(:read_with_stderr)
           spawn_opts[:err] = IO::NULL if stderr == :discard
+          if !mode.include?(:read) && !mode.include?(:read_with_stderr)
+            spawn_opts[:out] = IO::NULL
+          end
           writeio, readio, wait_thread = *Open3.popen2(*spawn_args, spawn_opts)
         elsif mode.include?(:read_with_stderr)
           writeio, readio, wait_thread = *Open3.popen2e(*spawn_args, spawn_opts)
@@ -275,8 +278,6 @@ module Fluent
         if mode.include?(:read) || mode.include?(:read_with_stderr)
           readio.set_encoding(external_encoding, internal_encoding, **encoding_options)
           readio_in_use = true
-        else
-          readio.reopen(IO::NULL) if readio
         end
         if mode.include?(:stderr)
           stderrio.set_encoding(external_encoding, internal_encoding, **encoding_options)

data/lib/fluent/plugin_helper/record_accessor.rb

@@ -41,9 +41,11 @@ module Fluent
             else
               mcall = method(:call_dig)
               mdelete = method(:delete_nest)
+              mset = method(:set_nest)
               singleton_class.module_eval do
                 define_method(:call, mcall)
                 define_method(:delete, mdelete)
+                define_method(:set, mset)
               end
             end
           end
@@ -75,6 +77,18 @@ module Fluent
           nil
         end
 
+        def set(r, v)
+          r[@keys] = v
+        end
+
+        # set_nest doesn't create intermediate object. If key doesn't exist, no effect.
+        # See also: https://bugs.ruby-lang.org/issues/11747
+        def set_nest(r, v)
+          r.dig(*@dig_keys)&.[]=(@last_key, v)
+        rescue
+          nil
+        end
+
         def self.parse_parameter(param)
           if param.start_with?('$.')
             parse_dot_notation(param)

data/lib/fluent/plugin_helper/server.rb

@@ -243,7 +243,7 @@ module Fluent
         :protocol, :version, :min_version, :max_version, :ciphers, :insecure,
         :ca_path, :cert_path, :private_key_path, :private_key_passphrase, :client_cert_auth,
         :ca_cert_path, :ca_private_key_path, :ca_private_key_passphrase,
-        :generate_private_key_length,
+        :cert_verifier, :generate_private_key_length,
         :generate_cert_country, :generate_cert_state, :generate_cert_state,
         :generate_cert_locality, :generate_cert_common_name,
         :generate_cert_expiration, :generate_cert_digest,
@@ -281,6 +281,8 @@ module Fluent
           config_param :ca_private_key_path, :string, default: nil
           config_param :ca_private_key_passphrase, :string, default: nil, secret: true
 
+          config_param :cert_verifier, :string, default: nil
+
           # Options for generating certs by private CA certs or self-signed
           config_param :generate_private_key_length, :integer, default: 2048
           config_param :generate_cert_country, :string, default: 'US'

data/lib/fluent/plugin_helper/service_discovery.rb

@@ -43,6 +43,13 @@ module Fluent
         super
       end
 
+      %i[after_start stop before_shutdown shutdown after_shutdown close terminate].each do |mth|
+        define_method(mth) do
+          @discovery_manager&.__send__(mth)
+          super()
+        end
+      end
+
       private
 
       # @param title [Symbol] the thread name. this value should be unique.

data/lib/fluent/plugin_helper/service_discovery/manager.rb

@@ -60,6 +60,14 @@ module Fluent
           end
         end
 
+        %i[after_start stop before_shutdown shutdown after_shutdown close terminate].each do |mth|
+          define_method(mth) do
+            @discoveries.each do |d|
+              d.__send__(mth)
+            end
+          end
+        end
+
         def run_once
           # Don't care race in this loop intentionally
           s = @queue.size

data/lib/fluent/plugin_helper/socket.rb

@@ -145,9 +145,16 @@ module Fluent
           context.ciphers = ciphers
           context.verify_mode = OpenSSL::SSL::VERIFY_PEER
           context.cert_store = cert_store
-          context.verify_hostname = true if verify_fqdn && fqdn && context.respond_to?(:verify_hostname=)
-          context.cert = OpenSSL::X509::Certificate.new(File.read(cert_path)) if cert_path
+          context.verify_hostname = verify_fqdn && fqdn
           context.key = OpenSSL::PKey::read(File.read(private_key_path), private_key_passphrase) if private_key_path
+
+          if cert_path
+            certs = socket_certificates_from_file(cert_path)
+            context.cert = certs.shift
+            unless certs.empty?
+              context.extra_chain_cert = certs
+            end
+          end
         end
         Fluent::TLS.set_version_to_context(context, version, min_version, max_version)
 
@@ -186,6 +193,17 @@ module Fluent
         end
       end
 
+      def socket_certificates_from_file(path)
+        data = File.read(path)
+        pattern = Regexp.compile('-+BEGIN CERTIFICATE-+\r?\n(?:[^-]*\r?\n)+-+END CERTIFICATE-+\r?\n?', Regexp::MULTILINE)
+        list = []
+        data.scan(pattern) { |match| list << OpenSSL::X509::Certificate.new(match) }
+        if list.length == 0
+          log.warn "cert_path does not contain a valid certificate"
+        end
+        list
+      end
+
       def self.tls_verify_result_name(code)
         case code
         when OpenSSL::X509::V_OK then 'V_OK'

data/lib/fluent/plugin_helper/socket_option.rb

@@ -21,6 +21,8 @@ require 'fcntl'
 module Fluent
   module PluginHelper
     module SocketOption
+      # ref: https://docs.microsoft.com/en-us/windows/win32/api/winsock/ns-winsock-linger
+      FORMAT_STRUCT_LINGER_WINDOWS  = 'S!S!' # { u_short l_onoff; u_short l_linger; }
       FORMAT_STRUCT_LINGER  = 'I!I!' # { int l_onoff; int l_linger; }
       FORMAT_STRUCT_TIMEVAL = 'L!L!' # { time_t tv_sec; suseconds_t tv_usec; }
 
@@ -49,9 +51,25 @@ module Fluent
         if nonblock
           sock.fcntl(Fcntl::F_SETFL, Fcntl::O_NONBLOCK)
         end
-        if linger_timeout
-          optval = [1, linger_timeout.to_i].pack(FORMAT_STRUCT_LINGER)
-          socket_option_set_one(sock, :SO_LINGER, optval)
+        if Fluent.windows?
+          # To prevent closing socket forcibly on Windows,
+          # this options shouldn't be set up when linger_timeout equals to 0 (including nil).
+          # This unintended behavior always ocurrs on Windows when linger_timeout.to_i == 0.
+          # This unintented behavior causes "Errno::ECONNRESET: An existing connection was forcibly
+          # closed by the remote host." on Windows.
+          if linger_timeout.to_i > 0
+            if linger_timeout >= 2**16
+              log.warn "maximum linger_timeout is 65535(2^16 - 1). Set to 65535 forcibly."
+              linger_timeout = 2**16 - 1
+            end
+            optval = [1, linger_timeout.to_i].pack(FORMAT_STRUCT_LINGER_WINDOWS)
+            socket_option_set_one(sock, :SO_LINGER, optval)
+          end
+        else
+          if linger_timeout
+            optval = [1, linger_timeout.to_i].pack(FORMAT_STRUCT_LINGER)
+            socket_option_set_one(sock, :SO_LINGER, optval)
+          end
         end
         if recv_timeout
           optval = [recv_timeout.to_i, 0].pack(FORMAT_STRUCT_TIMEVAL)

data/lib/fluent/supervisor.rb

@@ -301,6 +301,7 @@ module Fluent
 
       log_level = params['log_level']
       suppress_repeated_stacktrace = params['suppress_repeated_stacktrace']
+      ignore_repeated_log_interval = params['ignore_repeated_log_interval']
 
       log_path = params['log_path']
       chuser = params['chuser']
@@ -308,7 +309,7 @@ module Fluent
       log_rotate_age = params['log_rotate_age']
       log_rotate_size = params['log_rotate_size']
 
-      log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace}
+      log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace, ignore_repeated_log_interval: ignore_repeated_log_interval}
       logger_initializer = Supervisor::LoggerInitializer.new(
         log_path, log_level, chuser, chgroup, log_opts,
         log_rotate_age: log_rotate_age,
@@ -345,6 +346,7 @@ module Fluent
         chgroup: chgroup,
         chumask: 0,
         suppress_repeated_stacktrace: suppress_repeated_stacktrace,
+        ignore_repeated_log_interval: ignore_repeated_log_interval,
         daemonize: daemonize,
         rpc_endpoint: params['rpc_endpoint'],
         counter_server: params['counter_server'],
@@ -439,9 +441,10 @@ module Fluent
         self
       end
 
-      def apply_options(format: nil, time_format: nil, log_dir_perm: nil)
+      def apply_options(format: nil, time_format: nil, log_dir_perm: nil, ignore_repeated_log_interval: nil)
         $log.format = format if format
         $log.time_format = time_format if time_format
+        $log.ignore_repeated_log_interval = ignore_repeated_log_interval if ignore_repeated_log_interval
 
         if @path && log_dir_perm
           File.chmod(log_dir_perm || 0755, File.dirname(@path))
@@ -468,6 +471,7 @@ module Fluent
         root_dir: nil,
         suppress_interval: 0,
         suppress_repeated_stacktrace: true,
+        ignore_repeated_log_interval: nil,
         without_source: nil,
         use_v1_config: true,
         strict_config_value: nil,
@@ -507,7 +511,7 @@ module Fluent
       @cl_opt = opt
       @conf = nil
 
-      log_opts = { suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace] }
+      log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval]}
       @log = LoggerInitializer.new(
         @log_path, opt[:log_level], @chuser, @chgroup, log_opts,
         log_rotate_age: @log_rotate_age,
@@ -552,7 +556,7 @@ module Fluent
       end
 
       if dry_run
-        $log.info 'finsihed dry run mode'
+        $log.info 'finished dry run mode'
         exit 0
       else
         supervise
@@ -589,7 +593,10 @@ module Fluent
 
       main_process do
         create_socket_manager if @standalone_worker
-        ServerEngine::Privilege.change(@chuser, @chgroup) if @standalone_worker
+        if @standalone_worker
+          ServerEngine::Privilege.change(@chuser, @chgroup)
+          File.umask(0)
+        end
         MessagePackFactory.init(enable_time_support: @system_config.enable_msgpack_time_support)
         Fluent::Engine.init(@system_config)
         Fluent::Engine.run_configure(@conf)
@@ -628,6 +635,7 @@ module Fluent
         format: @system_config.log.format,
         time_format: @system_config.log.time_format,
         log_dir_perm: @system_config.dir_permission,
+        ignore_repeated_log_interval: @system_config.ignore_repeated_log_interval
       )
 
       $log.info :supervisor, 'parsing config file is succeeded', path: @config_path
@@ -690,6 +698,7 @@ module Fluent
         'root_dir' => @system_config.root_dir,
         'log_level' => @system_config.log_level,
         'suppress_repeated_stacktrace' => @system_config.suppress_repeated_stacktrace,
+        'ignore_repeated_log_interval' => @system_config.ignore_repeated_log_interval,
         'rpc_endpoint' => @system_config.rpc_endpoint,
         'enable_get_dump' => @system_config.enable_get_dump,
         'counter_server' => @system_config.counter_server,
@@ -882,7 +891,11 @@ module Fluent
     RUBY_ENCODING_OPTIONS_REGEX = %r{\A(-E|--encoding=|--internal-encoding=|--external-encoding=)}.freeze
 
     def build_spawn_command
-      fluentd_spawn_cmd = [ServerEngine.ruby_bin_path]
+      if ENV['TEST_RUBY_PATH']
+        fluentd_spawn_cmd = [ENV['TEST_RUBY_PATH']]
+      else
+        fluentd_spawn_cmd = [ServerEngine.ruby_bin_path]
+      end
 
       rubyopt = ENV['RUBYOPT']
       if rubyopt
@@ -897,10 +910,9 @@ module Fluent
 
       # Adding `-h` so that it can avoid ruby's command blocking
       # e.g. `ruby -Eascii-8bit:ascii-8bit` will block. but `ruby -Eascii-8bit:ascii-8bit -h` won't.
-      cmd = fluentd_spawn_cmd.join(' ')
-      _, e, s = Open3.capture3("#{cmd} -h")
+      _, e, s = Open3.capture3(*fluentd_spawn_cmd, "-h")
       if s.exitstatus != 0
-        $log.error('Invalid option is passed to RUBYOPT', command: cmd, error: e)
+        $log.error('Invalid option is passed to RUBYOPT', command: fluentd_spawn_cmd, error: e)
         exit s.exitstatus
       end