fluentd 0.14.11 → 0.14.12

data/lib/fluent/system_config.rb

@@ -22,23 +22,24 @@ module Fluent
     include Configurable
 
     SYSTEM_CONFIG_PARAMETERS = [
-      :root_dir, :log_level,
+      :workers, :root_dir, :log_level,
       :suppress_repeated_stacktrace, :emit_error_log_interval, :suppress_config_dump,
+      :log_event_verbose,
       :without_source, :rpc_endpoint, :enable_get_dump, :process_name,
       :file_permission, :dir_permission,
     ]
 
-    config_param :root_dir, :string, default: nil
-    config_param :log_level, default: nil do |level|
-      Log.str_to_level(level)
-    end
+    config_param :workers,   :integer, default: 1
+    config_param :root_dir,  :string, default: nil
+    config_param :log_level, :enum, list: [:trace, :debug, :info, :warn, :error, :fatal], default: nil
     config_param :suppress_repeated_stacktrace, :bool, default: nil
-    config_param :emit_error_log_interval, :time, default: nil
+    config_param :emit_error_log_interval,      :time, default: nil
     config_param :suppress_config_dump, :bool, default: nil
-    config_param :without_source, :bool, default: nil
-    config_param :rpc_endpoint, :string, default: nil
+    config_param :log_event_verbose,    :bool, default: nil
+    config_param :without_source,  :bool, default: nil
+    config_param :rpc_endpoint,    :string, default: nil
     config_param :enable_get_dump, :bool, default: nil
-    config_param :process_name, default: nil
+    config_param :process_name,    :string, default: nil
     config_param :file_permission, default: nil do |v|
       v.to_i(8)
     end
@@ -74,6 +75,12 @@ module Fluent
       configure(conf)
     end
 
+    def configure(conf)
+      super
+
+      @log_level = Log.str_to_level(@log_level.to_s) if @log_level
+    end
+
     def dup
       s = SystemConfig.new
       SYSTEM_CONFIG_PARAMETERS.each do |param|
@@ -82,11 +89,31 @@ module Fluent
       s
     end
 
+    def attach(supervisor)
+      system = self
+      supervisor.instance_eval {
+        SYSTEM_CONFIG_PARAMETERS.each do |param|
+          case param
+          when :rpc_endpoint, :enable_get_dump, :process_name, :file_permission, :dir_permission
+            next # doesn't exist in command line options
+          when :emit_error_log_interval
+            system.emit_error_log_interval = @suppress_interval if @suppress_interval
+          else
+            next unless instance_variable_defined?("@#{param}")
+            supervisor_value = instance_variable_get("@#{param}")
+            next if supervisor_value.nil? # it's not configured by command line options
+
+            system.send("#{param}=", supervisor_value)
+          end
+        end
+      }
+    end
+
     def apply(supervisor)
       system = self
       supervisor.instance_eval {
         SYSTEM_CONFIG_PARAMETERS.each do |param|
-          param_value = system.send(param)
+          param_value = system.__send__(param)
           next if param_value.nil?
 
           case param

data/lib/fluent/test.rb

@@ -36,6 +36,8 @@ module Fluent
     end
 
     def self.setup
+      ENV['SERVERENGINE_WORKER_ID'] = '0'
+
       $log = dummy_logger
 
       Fluent.__send__(:remove_const, :Engine)

data/lib/fluent/test/driver/base.rb

@@ -17,6 +17,7 @@
 require 'fluent/config'
 require 'fluent/config/element'
 require 'fluent/log'
+require 'fluent/clock'
 
 require 'serverengine/socket_manager'
 require 'fileutils'
@@ -48,8 +49,6 @@ module Fluent
 
           @logs = []
 
-          @test_clock_id = Process::CLOCK_MONOTONIC_RAW rescue Process::CLOCK_MONOTONIC
-
           @run_post_conditions = []
           @run_breaking_conditions = []
           @broken = false
@@ -77,8 +76,8 @@ module Fluent
           instance_start if start
 
           timeout ||= DEFAULT_TIMEOUT
-          stop_at = Process.clock_gettime(@test_clock_id) + timeout
-          @run_breaking_conditions << ->(){ Process.clock_gettime(@test_clock_id) >= stop_at }
+          stop_at = Fluent::Clock.now + timeout
+          @run_breaking_conditions << ->(){ Fluent::Clock.now >= stop_at }
 
           if !block_given? && @run_post_conditions.empty? && @run_breaking_conditions.empty?
             raise ArgumentError, "no stop conditions nor block specified"
@@ -139,38 +138,37 @@ module Fluent
         def instance_shutdown
           instance_hook_before_stopped
 
-          unless @instance.stopped?
-            @instance.stop rescue nil
-          end
-          unless @instance.before_shutdown?
-            @instance.before_shutdown rescue nil
-          end
-          unless @instance.shutdown?
-            @instance.shutdown rescue nil
+          show_errors_if_exists = ->(label, block){
+            begin
+              block.call
+            rescue => e
+              puts "unexpected error while #{label}, #{e.class}:#{e.message}"
+              e.backtrace.each do |bt|
+                puts "\t#{bt}"
+              end
+            end
+          }
+
+          show_errors_if_exists.call(:stop,            ->(){ @instance.stop unless @instance.stopped? })
+          show_errors_if_exists.call(:before_shutdown, ->(){ @instance.before_shutdown unless @instance.before_shutdown? })
+          show_errors_if_exists.call(:shutdown,        ->(){ @instance.shutdown unless @instance.shutdown? })
+          show_errors_if_exists.call(:after_shutdown,  ->(){ @instance.after_shutdown unless @instance.after_shutdown? })
+
+          if @instance.respond_to?(:server_wait_until_stop)
+            @instance.server_wait_until_stop
           end
 
           if @instance.respond_to?(:event_loop_wait_until_stop)
             @instance.event_loop_wait_until_stop
           end
 
-          unless @instance.after_shutdown?
-            @instance.after_shutdown rescue nil
-          end
-          unless @instance.closed?
-            @instance.close rescue nil
-          end
+          show_errors_if_exists.call(:close, ->(){ @instance.close unless @instance.closed? })
 
           if @instance.respond_to?(:thread_wait_until_stop)
             @instance.thread_wait_until_stop
           end
 
-          if @instance.respond_to?(:server_wait_until_stop)
-            @instance.server_wait_until_stop
-          end
-
-          unless @instance.terminated?
-            @instance.terminate rescue nil
-          end
+          show_errors_if_exists.call(:terminate, ->(){ @instance.terminate unless @instance.terminated? })
 
           if @socket_manager_server
             @socket_manager_server.close
@@ -195,7 +193,7 @@ module Fluent
 
           return_value = nil
           begin
-            Timeout.timeout(timeout * 1.1) do |sec|
+            Timeout.timeout(timeout * 2) do |sec|
               return_value = block.call if block_given?
             end
           rescue Timeout::Error

data/lib/fluent/test/helpers.rb

@@ -57,6 +57,27 @@ EOT
         ENV['TZ'] = oldtz
       end
 
+      def with_worker_config(root_dir: nil, workers: nil, worker_id: nil, &block)
+        if workers
+          if worker_id
+            if worker_id >= workers
+              raise "worker_id must be between 0 and (workers - 1)"
+            end
+          else
+            worker_id = 0
+          end
+        end
+
+        opts = {}
+        opts['root_dir'] = root_dir if root_dir
+        opts['workers'] = workers if workers
+
+        ENV['SERVERENGINE_WORKER_ID'] = worker_id.to_s
+        Fluent::SystemConfig.overwrite_system_config(opts, &block)
+      ensure
+        ENV.delete('SERVERENGINE_WORKER_ID')
+      end
+
       def time2str(time, localtime: false, format: nil)
         if format
           if localtime

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.14.11'
+  VERSION = '0.14.12'
 
 end

data/test/command/test_fluentd.rb

@@ -81,7 +81,7 @@ class TestFluentdCommand < ::Test::Unit::TestCase
     null_stream.close rescue nil
   end
 
-  def assert_log_matches(cmdline, *pattern_list, timeout: 10)
+  def assert_log_matches(cmdline, *pattern_list, patterns_not_match: [], timeout: 10)
     matched = false
     assert_error_msg = "matched correctly"
     stdio_buf = ""
@@ -118,6 +118,18 @@ class TestFluentdCommand < ::Test::Unit::TestCase
       assert_error_msg = "unexpected error in launching fluentd: #{e.inspect}\n" + stdio_buf
     end
     assert matched, assert_error_msg
+
+    unless patterns_not_match.empty?
+      lines = stdio_buf.split("\n")
+      patterns_not_match.each do |ptn|
+        matched_wrongly = if ptn.is_a? Regexp
+                            lines.any?{|line| ptn.match(line) }
+                          else
+                            lines.any?{|line| line.include?(ptn) }
+                          end
+        assert_false matched_wrongly, "pattern exists in logs wrongly:\n" + stdio_buf
+      end
+    end
   end
 
   def assert_fluentd_fails_to_start(cmdline, *pattern_list, timeout: 10)
@@ -187,7 +199,7 @@ CONF
       conf_path = create_conf_file('valid.conf', conf)
       assert File.exist?(conf_path)
 
-      assert_log_matches(create_cmdline(conf_path), "fluentd worker is now running", 'worker="0"')
+      assert_log_matches(create_cmdline(conf_path), "fluentd worker is now running", 'worker=0')
     end
   end
 
@@ -220,14 +232,14 @@ CONF
       conf_path = create_conf_file('existing_root_dir.conf', @conf)
       assert Dir.exist?(@root_path)
 
-      assert_log_matches(create_cmdline(conf_path), "fluentd worker is now running", 'worker="0"')
+      assert_log_matches(create_cmdline(conf_path), "fluentd worker is now running", 'worker=0')
     end
 
     test 'creates the specified root directory if missing' do
       conf_path = create_conf_file('missing_root_dir.conf', @conf)
       assert_false Dir.exist?(@root_path)
 
-      assert_log_matches(create_cmdline(conf_path), "fluentd worker is now running", 'worker="0"')
+      assert_log_matches(create_cmdline(conf_path), "fluentd worker is now running", 'worker=0')
       assert Dir.exist?(@root_path)
     end
 
@@ -244,6 +256,135 @@ CONF
     end
   end
 
+  sub_test_case 'configured to route log events to plugins' do
+    setup do
+      @basic_conf = <<CONF
+<source>
+  @type dummy
+  @id dummy
+  tag dummy
+  dummy {"message": "yay!"}
+</source>
+<match dummy>
+  @type null
+  @id   blackhole
+</match>
+CONF
+    end
+
+    test 'by top level <match fluent.*> section' do
+      conf = @basic_conf + <<CONF
+<match fluent.**>
+  @type stdout
+</match>
+CONF
+      conf_path = create_conf_file('logevent_1.conf', conf)
+      assert_log_matches(
+        create_cmdline(conf_path),
+        "fluentd worker is now running",
+        'fluent.info: {"worker":0,"message":"fluentd worker is now running worker=0"}',
+        patterns_not_match: ['[warn]: some tags for log events are not defined (to be ignored) tags=["fluent.trace", "fluent.debug"]'],
+      )
+    end
+
+    test 'by top level <match> section with warning for missing log levels (and warnings for each log event records)' do
+      conf = @basic_conf + <<CONF
+<match fluent.warn fluent.error fluent.fatal>
+  @type stdout
+</match>
+CONF
+      conf_path = create_conf_file('logevent_2.conf', conf)
+      assert_log_matches(
+        create_cmdline(conf_path),
+        "fluentd worker is now running",
+        '[warn]: match for some tags of log events are not defined (to be ignored) tags=["fluent.trace", "fluent.debug", "fluent.info"]',
+        '[warn]: #0 no patterns matched tag="fluent.info"',
+      )
+    end
+
+    test 'by <label @FLUENT_LOG> section' do
+      conf = @basic_conf + <<CONF
+<label @FLUENT_LOG>
+  <match **>
+    @type stdout
+  </match>
+</label>
+CONF
+      conf_path = create_conf_file('logevent_3.conf', conf)
+      assert_log_matches(
+        create_cmdline(conf_path),
+        "fluentd worker is now running",
+        'fluent.info: {"worker":0,"message":"fluentd worker is now running worker=0"}',
+        patterns_not_match: ['[warn]: some tags for log events are not defined (to be ignored)'],
+      )
+    end
+
+    test 'by <label> section with warning for missing log levels' do
+      conf = @basic_conf + <<CONF
+<label @FLUENT_LOG>
+  <match fluent.{trace,debug}>
+    @type null
+  </match>
+  <match fluent.warn fluent.error>
+    @type stdout
+  </match>
+</label>
+CONF
+      conf_path = create_conf_file('logevent_4.conf', conf)
+      assert_log_matches(
+        create_cmdline(conf_path),
+        "fluentd worker is now running",
+        '[warn]: match for some tags of log events are not defined (to be ignored) tags=["fluent.info", "fluent.fatal"]',
+        patterns_not_match: ['[warn]: no patterns matched tag="fluent.info"'],
+      )
+    end
+  end
+
+  sub_test_case 'configured to suppress configration dump' do
+    setup do
+      @basic_conf = <<CONF
+<source>
+  @type dummy
+  @id dummy
+  @label @dummydata
+  tag dummy
+  dummy {"message": "yay!"}
+</source>
+<label @dummydata>
+  <match dummy>
+    @type null
+    @id   blackhole
+  </match>
+</label>
+CONF
+    end
+
+    test 'configured by system config' do
+      conf = <<SYSTEM + @basic_conf
+<system>
+  suppress_config_dump
+</system>
+SYSTEM
+      conf_path = create_conf_file('suppress_conf_dump_1.conf', conf)
+      assert_log_matches(create_cmdline(conf_path), "fluentd worker is now running", patterns_not_match: ["tag dummy"])
+    end
+
+    test 'configured by command line option' do
+      conf_path = create_conf_file('suppress_conf_dump_2.conf', @basic_conf)
+      assert_log_matches(create_cmdline(conf_path, '--suppress-config-dump'), "fluentd worker is now running", patterns_not_match: ["tag dummy"])
+    end
+
+    test 'configured as false by system config, but overridden as true by command line option' do
+      conf = <<SYSTEM + @basic_conf
+<system>
+  suppress_config_dump false
+</system>
+SYSTEM
+      conf_path = create_conf_file('suppress_conf_dump_3.conf', conf)
+      assert_log_matches(create_cmdline(conf_path, '--suppress-config-dump'), "fluentd worker is now running", patterns_not_match: ["tag dummy"])
+    end
+  end
+
   sub_test_case 'configuration with wrong plugin type' do
     test 'failed to start' do
       conf = <<CONF
@@ -345,4 +486,133 @@ CONF
       )
     end
   end
+
+  sub_test_case 'configured to run 2 workers' do
+    setup do
+      @root_path = File.join(TMP_DIR, "rootpath")
+      FileUtils.rm_rf(@root_path)
+      FileUtils.mkdir_p(@root_path)
+    end
+
+    test 'success to start the number of workers specified in configuration' do
+      conf = <<CONF
+<system>
+  workers 2
+  root_dir #{@root_path}
+</system>
+<source>
+  @type dummy
+  @id dummy
+  @label @dummydata
+  tag dummy
+  dummy {"message": "yay!"}
+</source>
+<label @dummydata>
+  <match dummy>
+    @type null
+    @id   blackhole
+  </match>
+</label>
+CONF
+      conf_path = create_conf_file('workers1.conf', conf)
+      assert Dir.exist?(@root_path)
+
+      assert_log_matches(
+        create_cmdline(conf_path),
+        "#0 fluentd worker is now running worker=0",
+        "#1 fluentd worker is now running worker=1"
+      )
+    end
+
+    test 'success to start the number of workers specified by command line option' do
+      conf = <<CONF
+<system>
+  root_dir #{@root_path}
+</system>
+<source>
+  @type dummy
+  @id dummy
+  @label @dummydata
+  tag dummy
+  dummy {"message": "yay!"}
+</source>
+<label @dummydata>
+  <match dummy>
+    @type null
+    @id   blackhole
+  </match>
+</label>
+CONF
+      conf_path = create_conf_file('workers2.conf', conf)
+      assert_log_matches(
+        create_cmdline(conf_path, '--workers', '2'),
+        "#0 fluentd worker is now running worker=0",
+        "#1 fluentd worker is now running worker=1"
+      )
+    end
+
+    test 'failed to start workers when configured plugins do not support multi worker configuration' do
+      script =  "require 'fluent/plugin/input'\n"
+      script << "module Fluent::Plugin\n"
+      script << "  class SingleInput < Input\n"
+      script << "    Fluent::Plugin.register_input('single', self)\n"
+      script << "    def multi_workers_ready?\n"
+      script << "      false\n"
+      script << "    end\n"
+      script << "  end\n"
+      script << "end\n"
+      plugin_path = create_plugin_file('in_single.rb', script)
+
+      conf = <<CONF
+<system>
+  workers 2
+</system>
+<source>
+  @type single
+  @id single
+  @label @dummydata
+</source>
+<label @dummydata>
+  <match dummy>
+    @type null
+    @id   blackhole
+  </match>
+</label>
+CONF
+      conf_path = create_conf_file('workers_invalid1.conf', conf)
+      assert_fluentd_fails_to_start(
+        create_cmdline(conf_path, "-p", File.dirname(plugin_path)),
+        "Plugin 'single' does not support multi workers configuration (Fluent::Plugin::SingleInput)",
+      )
+    end
+
+    test 'failed to start workers when file buffer is configured in non-workers way' do
+      conf = <<CONF
+<system>
+  workers 2
+</system>
+<source>
+  @type single
+  @id single
+  @label @dummydata
+</source>
+<label @dummydata>
+  <match dummy>
+    @type null
+    @id   blackhole
+    <buffer>
+      @type file
+      path #{File.join(@root_path, "buf", "file.*.log")}
+    </buffer>
+  </match>
+</label>
+CONF
+      conf_path = create_conf_file('workers_invalid2.conf', conf)
+      assert_fluentd_fails_to_start(
+        create_cmdline(conf_path),
+        "[blackhole] file buffer with multi workers should be configured to use directory 'path', or system root_dir and plugin id",
+        "config error file=\"#{conf_path}\" error_class=Fluent::ConfigError error=\"Plugin 'file' does not support multi workers configuration (Fluent::Plugin::FileBuffer)\"",
+      )
+    end
+  end
 end