fluentd 1.10.1 → 1.11.1

data/lib/fluent/plugin_helper/service_discovery/manager.rb

@@ -60,6 +60,14 @@ module Fluent
           end
         end
 
+        %i[after_start stop before_shutdown shutdown after_shutdown close terminate].each do |mth|
+          define_method(mth) do
+            @discoveries.each do |d|
+              d.__send__(mth)
+            end
+          end
+        end
+
         def run_once
           # Don't care race in this loop intentionally
           s = @queue.size

data/lib/fluent/plugin_helper/socket.rb

@@ -145,9 +145,16 @@ module Fluent
           context.ciphers = ciphers
           context.verify_mode = OpenSSL::SSL::VERIFY_PEER
           context.cert_store = cert_store
-          context.verify_hostname = true if verify_fqdn && fqdn && context.respond_to?(:verify_hostname=)
-          context.cert = OpenSSL::X509::Certificate.new(File.read(cert_path)) if cert_path
+          context.verify_hostname = verify_fqdn && fqdn
           context.key = OpenSSL::PKey::read(File.read(private_key_path), private_key_passphrase) if private_key_path
+
+          if cert_path
+            certs = socket_certificates_from_file(cert_path)
+            context.cert = certs.shift
+            unless certs.empty?
+              context.extra_chain_cert = certs
+            end
+          end
         end
         Fluent::TLS.set_version_to_context(context, version, min_version, max_version)
 
@@ -186,6 +193,17 @@ module Fluent
         end
       end
 
+      def socket_certificates_from_file(path)
+        data = File.read(path)
+        pattern = Regexp.compile('-+BEGIN CERTIFICATE-+\r?\n(?:[^-]*\r?\n)+-+END CERTIFICATE-+\r?\n?', Regexp::MULTILINE)
+        list = []
+        data.scan(pattern) { |match| list << OpenSSL::X509::Certificate.new(match) }
+        if list.length == 0
+          log.warn "cert_path does not contain a valid certificate"
+        end
+        list
+      end
+
       def self.tls_verify_result_name(code)
         case code
         when OpenSSL::X509::V_OK then 'V_OK'

data/lib/fluent/plugin_helper/socket_option.rb

@@ -67,8 +67,8 @@ module Fluent
           end
         else
           if linger_timeout
-          optval = [1, linger_timeout.to_i].pack(FORMAT_STRUCT_LINGER)
-          socket_option_set_one(sock, :SO_LINGER, optval)
+            optval = [1, linger_timeout.to_i].pack(FORMAT_STRUCT_LINGER)
+            socket_option_set_one(sock, :SO_LINGER, optval)
           end
         end
         if recv_timeout

data/lib/fluent/supervisor.rb

@@ -301,6 +301,7 @@ module Fluent
 
       log_level = params['log_level']
       suppress_repeated_stacktrace = params['suppress_repeated_stacktrace']
+      ignore_repeated_log_interval = params['ignore_repeated_log_interval']
 
       log_path = params['log_path']
       chuser = params['chuser']
@@ -308,7 +309,7 @@ module Fluent
       log_rotate_age = params['log_rotate_age']
       log_rotate_size = params['log_rotate_size']
 
-      log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace}
+      log_opts = {suppress_repeated_stacktrace: suppress_repeated_stacktrace, ignore_repeated_log_interval: ignore_repeated_log_interval}
       logger_initializer = Supervisor::LoggerInitializer.new(
         log_path, log_level, chuser, chgroup, log_opts,
         log_rotate_age: log_rotate_age,
@@ -345,6 +346,7 @@ module Fluent
         chgroup: chgroup,
         chumask: 0,
         suppress_repeated_stacktrace: suppress_repeated_stacktrace,
+        ignore_repeated_log_interval: ignore_repeated_log_interval,
         daemonize: daemonize,
         rpc_endpoint: params['rpc_endpoint'],
         counter_server: params['counter_server'],
@@ -439,9 +441,10 @@ module Fluent
         self
       end
 
-      def apply_options(format: nil, time_format: nil, log_dir_perm: nil)
+      def apply_options(format: nil, time_format: nil, log_dir_perm: nil, ignore_repeated_log_interval: nil)
         $log.format = format if format
         $log.time_format = time_format if time_format
+        $log.ignore_repeated_log_interval = ignore_repeated_log_interval if ignore_repeated_log_interval
 
         if @path && log_dir_perm
           File.chmod(log_dir_perm || 0755, File.dirname(@path))
@@ -468,6 +471,7 @@ module Fluent
         root_dir: nil,
         suppress_interval: 0,
         suppress_repeated_stacktrace: true,
+        ignore_repeated_log_interval: nil,
         without_source: nil,
         use_v1_config: true,
         strict_config_value: nil,
@@ -507,7 +511,7 @@ module Fluent
       @cl_opt = opt
       @conf = nil
 
-      log_opts = { suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace] }
+      log_opts = {suppress_repeated_stacktrace: opt[:suppress_repeated_stacktrace], ignore_repeated_log_interval: opt[:ignore_repeated_log_interval]}
       @log = LoggerInitializer.new(
         @log_path, opt[:log_level], @chuser, @chgroup, log_opts,
         log_rotate_age: @log_rotate_age,
@@ -552,7 +556,7 @@ module Fluent
       end
 
       if dry_run
-        $log.info 'finsihed dry run mode'
+        $log.info 'finished dry run mode'
         exit 0
       else
         supervise
@@ -589,7 +593,10 @@ module Fluent
 
       main_process do
         create_socket_manager if @standalone_worker
-        ServerEngine::Privilege.change(@chuser, @chgroup) if @standalone_worker
+        if @standalone_worker
+          ServerEngine::Privilege.change(@chuser, @chgroup)
+          File.umask(0)
+        end
         MessagePackFactory.init(enable_time_support: @system_config.enable_msgpack_time_support)
         Fluent::Engine.init(@system_config)
         Fluent::Engine.run_configure(@conf)
@@ -628,6 +635,7 @@ module Fluent
         format: @system_config.log.format,
         time_format: @system_config.log.time_format,
         log_dir_perm: @system_config.dir_permission,
+        ignore_repeated_log_interval: @system_config.ignore_repeated_log_interval
       )
 
       $log.info :supervisor, 'parsing config file is succeeded', path: @config_path
@@ -690,6 +698,7 @@ module Fluent
         'root_dir' => @system_config.root_dir,
         'log_level' => @system_config.log_level,
         'suppress_repeated_stacktrace' => @system_config.suppress_repeated_stacktrace,
+        'ignore_repeated_log_interval' => @system_config.ignore_repeated_log_interval,
         'rpc_endpoint' => @system_config.rpc_endpoint,
         'enable_get_dump' => @system_config.enable_get_dump,
         'counter_server' => @system_config.counter_server,
@@ -882,7 +891,11 @@ module Fluent
     RUBY_ENCODING_OPTIONS_REGEX = %r{\A(-E|--encoding=|--internal-encoding=|--external-encoding=)}.freeze
 
     def build_spawn_command
-      fluentd_spawn_cmd = [ServerEngine.ruby_bin_path]
+      if ENV['TEST_RUBY_PATH']
+        fluentd_spawn_cmd = [ENV['TEST_RUBY_PATH']]
+      else
+        fluentd_spawn_cmd = [ServerEngine.ruby_bin_path]
+      end
 
       rubyopt = ENV['RUBYOPT']
       if rubyopt
@@ -897,10 +910,9 @@ module Fluent
 
       # Adding `-h` so that it can avoid ruby's command blocking
       # e.g. `ruby -Eascii-8bit:ascii-8bit` will block. but `ruby -Eascii-8bit:ascii-8bit -h` won't.
-      cmd = fluentd_spawn_cmd.join(' ')
-      _, e, s = Open3.capture3("#{cmd} -h")
+      _, e, s = Open3.capture3(*fluentd_spawn_cmd, "-h")
       if s.exitstatus != 0
-        $log.error('Invalid option is passed to RUBYOPT', command: cmd, error: e)
+        $log.error('Invalid option is passed to RUBYOPT', command: fluentd_spawn_cmd, error: e)
         exit s.exitstatus
       end
 

data/lib/fluent/system_config.rb

@@ -24,7 +24,7 @@ module Fluent
     SYSTEM_CONFIG_PARAMETERS = [
       :workers, :root_dir, :log_level,
       :suppress_repeated_stacktrace, :emit_error_log_interval, :suppress_config_dump,
-      :log_event_verbose,
+      :log_event_verbose, :ignore_repeated_log_interval,
       :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
       :file_permission, :dir_permission, :counter_server, :counter_client,
       :strict_config_value, :enable_msgpack_time_support
@@ -34,6 +34,7 @@ module Fluent
     config_param :root_dir,  :string, default: nil
     config_param :log_level, :enum, list: [:trace, :debug, :info, :warn, :error, :fatal], default: 'info'
     config_param :suppress_repeated_stacktrace, :bool, default: nil
+    config_param :ignore_repeated_log_interval, :time, default: nil
     config_param :emit_error_log_interval,      :time, default: nil
     config_param :suppress_config_dump, :bool, default: nil
     config_param :log_event_verbose,    :bool, default: nil

data/lib/fluent/test/filter_test.rb

@@ -30,12 +30,12 @@ module Fluent
       attr_reader :filtered
       attr_accessor :tag
 
-      def emit(record, time = Engine.now)
+      def emit(record, time = EventTime.now)
         emit_with_tag(@tag, record, time)
       end
       alias_method :filter, :emit
 
-      def emit_with_tag(tag, record, time = Engine.now)
+      def emit_with_tag(tag, record, time = EventTime.now)
         @events[tag] ||= MultiEventStream.new
         @events[tag].add(time, record)
       end

data/lib/fluent/test/output_test.rb

@@ -41,7 +41,7 @@ module Fluent
 
       attr_accessor :tag
 
-      def emit(record, time=Engine.now)
+      def emit(record, time=EventTime.now)
         es = OneEventStream.new(time, record)
         @instance.emit_events(@tag, es)
       end
@@ -62,7 +62,7 @@ module Fluent
 
       attr_accessor :tag
 
-      def emit(record, time=Engine.now)
+      def emit(record, time=EventTime.now)
         @entries << [time, record]
         self
       end
@@ -110,7 +110,7 @@ module Fluent
 
       attr_accessor :tag
 
-      def emit(record, time=Engine.now)
+      def emit(record, time=EventTime.now)
         @entries << [time, record]
         self
       end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.10.1'
+  VERSION = '1.11.1'
 
 end

data/test/command/test_fluentd.rb

@@ -16,6 +16,7 @@ class TestFluentdCommand < ::Test::Unit::TestCase
     FileUtils.mkdir_p(TMP_DIR)
     @supervisor_pid = nil
     @worker_pids = []
+    ENV["TEST_RUBY_PATH"] = nil
   end
 
   def process_exist?(pid)
@@ -98,7 +99,8 @@ class TestFluentdCommand < ::Test::Unit::TestCase
 
   def assert_log_matches(cmdline, *pattern_list, patterns_not_match: [], timeout: 10, env: {})
     matched = false
-    assert_error_msg = "matched correctly"
+    matched_wrongly = false
+    assert_error_msg = ""
     stdio_buf = ""
     begin
       execute_command(cmdline, TMP_DIR, env) do |pid, stdout|
@@ -128,13 +130,18 @@ class TestFluentdCommand < ::Test::Unit::TestCase
         end
       end
     rescue Timeout::Error
-      assert_error_msg = "execution timeout with command out:\n" + stdio_buf
+      assert_error_msg = "execution timeout"
     rescue => e
-      assert_error_msg = "unexpected error in launching fluentd: #{e.inspect}\n" + stdio_buf
+      assert_error_msg = "unexpected error in launching fluentd: #{e.inspect}"
+    else
+      assert_error_msg = "log doesn't match" unless matched
     end
-    assert matched, assert_error_msg
 
-    unless patterns_not_match.empty?
+    if patterns_not_match.empty?
+      assert_error_msg = build_message(assert_error_msg,
+                                       "<?>\nwas expected to include:\n<?>",
+                                       stdio_buf, pattern_list)
+    else
       lines = stdio_buf.split("\n")
       patterns_not_match.each do |ptn|
         matched_wrongly = if ptn.is_a? Regexp
@@ -142,9 +149,17 @@ class TestFluentdCommand < ::Test::Unit::TestCase
                           else
                             lines.any?{|line| line.include?(ptn) }
                           end
-        assert_false matched_wrongly, "pattern exists in logs wrongly:\n" + stdio_buf
+        if matched_wrongly
+          assert_error_msg << "\n" unless assert_error_msg.empty?
+          assert_error_msg << "pattern exists in logs wrongly: #{ptn}"
+        end
       end
+      assert_error_msg = build_message(assert_error_msg,
+                                       "<?>\nwas expected to include:\n<?>\nand not include:\n<?>",
+                                       stdio_buf, pattern_list, patterns_not_match)
     end
+
+    assert matched && !matched_wrongly, assert_error_msg
   end
 
   def assert_fluentd_fails_to_start(cmdline, *pattern_list, timeout: 10)
@@ -842,8 +857,7 @@ CONF
       '-external-encoding' => '--external-encoding=utf-8',
       '-internal-encoding' => '--internal-encoding=utf-8',
     )
-    test "-E option is set to RUBYOPT3" do |opt|
-      omit "hard to run correctly on Windows. Need to debug." if Fluent.windows?
+    test "-E option is set to RUBYOPT" do |opt|
       conf = <<CONF
 <source>
   @type dummy
@@ -854,6 +868,7 @@ CONF
 </match>
 CONF
       conf_path = create_conf_file('rubyopt_test.conf', conf)
+      opt << " #{ENV['RUBYOPT']}" if ENV['RUBYOPT']
       assert_log_matches(
         create_cmdline(conf_path),
         *opt.split(' '),
@@ -862,7 +877,7 @@ CONF
       )
     end
 
-        test "without RUBYOPT" do
+    test "without RUBYOPT" do
       conf = <<CONF
 <source>
   @type dummy
@@ -877,7 +892,7 @@ CONF
     end
 
     test 'invalid values are set to RUBYOPT' do
-      omit "hard to run correctly on Windows. Need to debug." if Fluent.windows?
+      omit "hard to run correctly because RUBYOPT=-r/path/to/bundler/setup is required on Windows while this test set invalid RUBYOPT" if Fluent.windows?
       conf = <<CONF
 <source>
   @type dummy
@@ -895,6 +910,38 @@ CONF
       )
     end
 
+    # https://github.com/fluent/fluentd/issues/2915
+    test "ruby path contains spaces" do
+      conf = <<CONF
+<source>
+  @type dummy
+  tag dummy
+</source>
+<match>
+  @type null
+</match>
+CONF
+      ruby_path = ServerEngine.ruby_bin_path
+      tmp_ruby_path = File.join(TMP_DIR, "ruby with spaces")
+      if Fluent.windows?
+        tmp_ruby_path << ".bat"
+        File.open(tmp_ruby_path, "w") do |file|
+          file.write "#{ruby_path} %*"
+        end
+      else
+        FileUtils.ln_sf(ruby_path, tmp_ruby_path)
+      end
+      ENV["TEST_RUBY_PATH"] = tmp_ruby_path
+      cmd_path = File.expand_path(File.dirname(__FILE__) + "../../../bin/fluentd")
+      conf_path = create_conf_file('space_mixed_ruby_path_test.conf', conf)
+      args = ["bundle", "exec", tmp_ruby_path, cmd_path, "-c", conf_path]
+      assert_log_matches(
+        args,
+        'spawn command to main:',
+        '-Eascii-8bit:ascii-8bit'
+      )
+    end
+
     test 'success to start workers when file buffer is configured in non-workers way only for specific worker' do
       conf = <<CONF
 <system>

data/test/config/test_system_config.rb

@@ -73,6 +73,7 @@ module Fluent::Config
       assert_nil(sc.root_dir)
       assert_equal(Fluent::Log::LEVEL_INFO, sc.log_level)
       assert_nil(sc.suppress_repeated_stacktrace)
+      assert_nil(sc.ignore_repeated_log_interval)
       assert_nil(sc.emit_error_log_interval)
       assert_nil(sc.suppress_config_dump)
       assert_nil(sc.without_source)
@@ -86,6 +87,7 @@ module Fluent::Config
       'root_dir' => ['root_dir', File.join(TMP_DIR, 'root')],
       'log_level' => ['log_level', 'error'],
       'suppress_repeated_stacktrace' => ['suppress_repeated_stacktrace', true],
+      'ignore_repeated_log_interval' => ['ignore_repeated_log_interval', 10],
       'log_event_verbose' => ['log_event_verbose', true],
       'suppress_config_dump' => ['suppress_config_dump', true],
       'without_source' => ['without_source', true],

data/test/plugin/out_forward/test_load_balancer.rb

@@ -49,6 +49,44 @@ class LoadBalancerTest < Test::Unit::TestCase
       end
     end
 
+    test 'call like round robin without weight=0 node' do
+      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
+      n1 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
+      n2 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 1)
+      n3 = flexmock('node', :'standby?' => false, :'available?' => true, weight: 0)
+
+      lb.rebuild_weight_array([n1, n2, n3])
+
+      lb.select_healthy_node do |node|
+        # to handle random choice
+        if node == n1
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+        else
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n2)
+          end
+
+          lb.select_healthy_node do |node|
+            assert_equal(node, n1)
+          end
+        end
+      end
+    end
+
     test 'raise an error if all node are unavialble' do
       lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
       lb.rebuild_weight_array([flexmock('node', :'standby?' => false, :'available?' => false, weight: 1)])
@@ -56,5 +94,13 @@ class LoadBalancerTest < Test::Unit::TestCase
         lb.select_healthy_node
       end
     end
+
+    test 'it regards weight=0 node as unavialble' do
+      lb = Fluent::Plugin::ForwardOutput::LoadBalancer.new($log)
+      lb.rebuild_weight_array([flexmock('node', :'standby?' => false, :'available?' => true, weight: 0)])
+      assert_raise(Fluent::Plugin::ForwardOutput::NoNodesAvailable) do
+        lb.select_healthy_node
+      end
+    end
   end
 end