fluentd 1.8.0.rc2 → 1.8.0.rc3

data/lib/fluent/system_config.rb

@@ -115,50 +115,12 @@ module Fluent
       s
     end
 
-    def attach(supervisor)
-      system = self
-      supervisor.instance_eval {
-        SYSTEM_CONFIG_PARAMETERS.each do |param|
-          case param
-          when :rpc_endpoint, :enable_get_dump, :process_name, :file_permission, :dir_permission, :counter_server, :counter_client
-            next # doesn't exist in command line options
-          when :emit_error_log_interval
-            system.emit_error_log_interval = @suppress_interval if @suppress_interval
-          when :log_level
-            ll_value = instance_variable_get("@log_level")
-            # info level can't be specified via command line option.
-            # log_level is info here, it is default value and <system>'s log_level should be applied if exists.
-            if ll_value != Fluent::Log::LEVEL_INFO
-              system.log_level = ll_value
-            end
-          else
-            next unless instance_variable_defined?("@#{param}")
-            supervisor_value = instance_variable_get("@#{param}")
-            next if supervisor_value.nil? # it's not configured by command line options
-
-            system.__send__("#{param}=", supervisor_value)
-          end
-        end
-      }
-    end
-
-    def apply(supervisor)
-      system = self
-      supervisor.instance_eval {
-        SYSTEM_CONFIG_PARAMETERS.each do |param|
-          param_value = system.__send__(param)
-          next if param_value.nil?
-
-          case param
-          when :log_level
-            @log.level = @log_level = param_value
-          else
-            instance_variable_set("@#{param}", param_value)
-          end
+    def overwrite_variables(**opt)
+      SYSTEM_CONFIG_PARAMETERS.each do |param|
+        if opt.key?(param) && !opt[param].nil? && instance_variable_defined?("@#{param}")
+          instance_variable_set("@#{param}", opt[param])
         end
-        #@counter_server = system.counter_server unless system.counter_server.nil?
-        #@counter_client = system.counter_client unless system.counter_client.nil?
-      }
+      end
     end
 
     module Mixin

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.8.0.rc2'
+  VERSION = '1.8.0.rc3'
 
 end

data/test/command/test_fluentd.rb

@@ -317,7 +317,8 @@ CONF
         create_cmdline(conf_path),
         "fluentd worker is now running",
         'fluent.info: {"worker":0,"message":"fluentd worker is now running worker=0"}',
-        patterns_not_match: ['[warn]: some tags for log events are not defined (to be ignored) tags=["fluent.trace", "fluent.debug"]'],
+        "define <match fluent.**> to capture fluentd logs in top level is deprecated. Use <label @FLUENT_LOG> instead",
+        patterns_not_match: ['[warn]: some tags for log events are not defined in top level (to be ignored) tags=["fluent.trace", "fluent.debug"]'],
       )
     end
 
@@ -331,7 +332,8 @@ CONF
       assert_log_matches(
         create_cmdline(conf_path),
         "fluentd worker is now running",
-        '[warn]: #0 match for some tags of log events are not defined (to be ignored) tags=["fluent.trace", "fluent.debug", "fluent.info"]',
+        '[warn]: #0 match for some tags of log events are not defined in top level (to be ignored) tags=["fluent.trace", "fluent.debug", "fluent.info"]',
+        "define <match fluent.warn>, <match fluent.error>, <match fluent.fatal> to capture fluentd logs in top level is deprecated. Use <label @FLUENT_LOG> instead",
         '[warn]: #0 no patterns matched tag="fluent.info"',
       )
     end
@@ -349,7 +351,7 @@ CONF
         create_cmdline(conf_path),
         "fluentd worker is now running",
         'fluent.info: {"worker":0,"message":"fluentd worker is now running worker=0"}',
-        patterns_not_match: ['[warn]: some tags for log events are not defined (to be ignored)'],
+        patterns_not_match: ['[warn]: some tags for log events are not defined in @FLUENT_LOG label (to be ignored)'],
       )
     end
 
@@ -368,7 +370,7 @@ CONF
       assert_log_matches(
         create_cmdline(conf_path),
         "fluentd worker is now running",
-        '[warn]: #0 match for some tags of log events are not defined (to be ignored) tags=["fluent.info", "fluent.fatal"]',
+        '[warn]: #0 match for some tags of log events are not defined in @FLUENT_LOG label (to be ignored) tags=["fluent.info", "fluent.fatal"]',
         patterns_not_match: ['[warn]: no patterns matched tag="fluent.info"'],
       )
     end
@@ -625,7 +627,8 @@ CONF
   workers 2
 </system>
 <source>
-  @type single
+  @type dummy
+  tag dummy
   @id single
   @label @dummydata
 </source>
@@ -669,7 +672,8 @@ EOC
   workers 2
 </system>
 <source>
-  @type single
+  @type dummy
+  tag dummy
   @id single
   @label @dummydata
 </source>
@@ -793,7 +797,7 @@ CONF
     @id   blackhole
     <buffer>
       @type file
-      path #{File.join(@root_path, "buf", "file.*.log")}
+      path #{File.join(@root_path, "buf")}
     </buffer>
   </match>
 </worker>
@@ -863,6 +867,7 @@ module Fluent::Plugin
   class FakeInput < Input
     Fluent::Plugin.register_input('fake', self)
     config_param :secret, :string, secret: true
+    def multi_workers_ready?; true; end
   end
 end
 EOC

data/test/config/test_system_config.rb

@@ -15,20 +15,41 @@ module Fluent::Config
   class FakeSupervisor
     attr_writer :log_level
 
-    def initialize
-      @workers = nil
-      @root_dir = nil
-      @log = FakeLoggerInitializer.new
-      @log_level = Fluent::Log::LEVEL_INFO
-      @suppress_interval = nil
-      @suppress_config_dump = nil
-      @suppress_repeated_stacktrace = nil
-      @log_event_label = nil
-      @log_event_verbose = nil
-      @without_source = nil
-      @emit_error_log_interval = nil
-      @file_permission = nil
-      @dir_permission = nil
+    def initialize(**opt)
+      @system_config = nil
+      @cl_opt = {
+        wokers: nil,
+        root_dir: nil,
+        log: FakeLoggerInitializer.new,
+        log_level: Fluent::Log::LEVEL_INFO,
+        suppress_interval: nil,
+        suppress_config_dump: nil,
+        suppress_repeated_stacktrace: nil,
+        log_event_label: nil,
+        log_event_verbose: nil,
+        without_source: nil,
+        emit_error_log_interval: nil,
+        file_permission: nil,
+        dir_permission: nil,
+      }.merge(opt)
+    end
+
+    def for_system_config
+      opt = {}
+      # this is copy from Supervisor#build_system_config
+      Fluent::SystemConfig::SYSTEM_CONFIG_PARAMETERS.each do |param|
+        if @cl_opt.key?(param) && !@cl_opt[param].nil?
+          if param == :log_level && @cl_opt[:log_level] == Fluent::Log::LEVEL_INFO
+            # info level can't be specified via command line option.
+            # log_level is info here, it is default value and <system>'s log_level should be applied if exists.
+            next
+          end
+
+          opt[param] = @cl_opt[param]
+        end
+      end
+
+      opt
     end
   end
 
@@ -47,7 +68,7 @@ module Fluent::Config
       EOS
       s = FakeSupervisor.new
       sc = Fluent::SystemConfig.new(conf)
-      sc.apply(s)
+      sc.overwrite_variables(s.for_system_config)
       assert_equal(1, sc.workers)
       assert_nil(sc.root_dir)
       assert_nil(sc.log_level)
@@ -57,16 +78,6 @@ module Fluent::Config
       assert_nil(sc.without_source)
       assert_equal(:text, sc.log.format)
       assert_equal('%Y-%m-%d %H:%M:%S %z', sc.log.time_format)
-      assert_equal(1, s.instance_variable_get(:@workers))
-      assert_nil(s.instance_variable_get(:@root_dir))
-      assert_equal(Fluent::Log::LEVEL_INFO, s.instance_variable_get(:@log_level))
-      assert_nil(s.instance_variable_get(:@suppress_repeated_stacktrace))
-      assert_nil(s.instance_variable_get(:@emit_error_log_interval))
-      assert_nil(s.instance_variable_get(:@suppress_config_dump))
-      assert_nil(s.instance_variable_get(:@log_event_verbose))
-      assert_nil(s.instance_variable_get(:@without_source))
-      assert_nil(s.instance_variable_get(:@file_permission))
-      assert_nil(s.instance_variable_get(:@dir_permission))
     end
 
     data(
@@ -86,10 +97,12 @@ module Fluent::Config
       EOS
       s = FakeSupervisor.new
       sc = Fluent::SystemConfig.new(conf)
-      sc.apply(s)
-      assert_not_nil(sc.instance_variable_get("@#{k}"))
-      key = (k == 'emit_error_log_interval' ? 'suppress_interval' : k)
-      assert_not_nil(s.instance_variable_get("@#{key}"))
+      sc.overwrite_variables(s.for_system_config)
+      if k == 'log_level'
+        assert_equal(Fluent::Log::LEVEL_ERROR, sc.__send__(k))
+      else
+        assert_equal(v, sc.__send__(k))
+      end
     end
 
     test "log parameters" do
@@ -103,50 +116,11 @@ module Fluent::Config
       EOS
       s = FakeSupervisor.new
       sc = Fluent::SystemConfig.new(conf)
-      sc.apply(s)
+      sc.overwrite_variables(s.for_system_config)
       assert_equal(:json, sc.log.format)
       assert_equal('%Y', sc.log.time_format)
     end
 
-    data(
-      'foo' => ['foo', 'bar'],
-      'hoge' => ['hoge', 'fuga'],
-    )
-    test "should not affect settable parameters with unknown parameters" do |(k, v)|
-      s = FakeSupervisor.new
-      sc = Fluent::SystemConfig.new({k => v})
-      sc.apply(s)
-      assert_equal(1, s.instance_variable_get(:@workers))
-      assert_nil(s.instance_variable_get(:@root_dir))
-      assert_equal(Fluent::Log::LEVEL_INFO, s.instance_variable_get(:@log_level))
-      assert_nil(s.instance_variable_get(:@suppress_repeated_stacktrace))
-      assert_nil(s.instance_variable_get(:@emit_error_log_interval))
-      assert_nil(s.instance_variable_get(:@suppress_config_dump))
-      assert_nil(s.instance_variable_get(:@log_event_verbose))
-      assert_nil(s.instance_variable_get(:@without_source))
-      assert_nil(s.instance_variable_get(:@file_permission))
-      assert_nil(s.instance_variable_get(:@dir_permission))
-    end
-
-    data('trace' => Fluent::Log::LEVEL_TRACE,
-         'debug' => Fluent::Log::LEVEL_DEBUG,
-         'info' => Fluent::Log::LEVEL_INFO,
-         'warn' => Fluent::Log::LEVEL_WARN,
-         'error' => Fluent::Log::LEVEL_ERROR,
-         'fatal' => Fluent::Log::LEVEL_FATAL)
-    test 'log_level is applied when log_level related command line option is not passed' do |level|
-      conf = parse_text(<<-EOS)
-        <system>
-          log_level #{Fluent::Log::LEVEL_TEXT[level]}
-        </system>
-      EOS
-      s = FakeSupervisor.new
-      sc = Fluent::SystemConfig.new(conf)
-      sc.attach(s)
-      sc.apply(s)
-      assert_equal(level, s.instance_variable_get("@log").level)
-    end
-
     # info is removed because info level can't be specified via command line
     data('trace' => Fluent::Log::LEVEL_TRACE,
          'debug' => Fluent::Log::LEVEL_DEBUG,
@@ -159,27 +133,10 @@ module Fluent::Config
           log_level info
         </system>
       EOS
-      s = FakeSupervisor.new
-      s.log_level = level
-      sc = Fluent::SystemConfig.new(conf)
-      sc.attach(s)
-      sc.apply(s)
-      assert_equal(level, s.instance_variable_get("@log").level)
-    end
-
-    test 'process global overridable variables' do
-      conf = parse_text(<<-EOS)
-        <system>
-          file_permission 0655
-          dir_permission 0765
-        </system>
-      EOS
-      s = FakeSupervisor.new
+      s = FakeSupervisor.new(log_level: level)
       sc = Fluent::SystemConfig.new(conf)
-      sc.attach(s)
-      sc.apply(s)
-      assert_equal(0655, s.instance_variable_get(:@file_permission))
-      assert_equal(0765, s.instance_variable_get(:@dir_permission))
+      sc.overwrite_variables(s.for_system_config)
+      assert_equal(level, sc.log_level)
     end
   end
 end

data/test/plugin/test_buf_file.rb

@@ -327,6 +327,46 @@ class FileBufferTest < Test::Unit::TestCase
       plugin.stop; plugin.before_shutdown; plugin.shutdown; plugin.after_shutdown; plugin.close; plugin.terminate
       FileUtils.rm_r bufdir
     end
+
+    test '#generate_chunk generates blank file chunk with specified permission with system_config' do
+      omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
+
+      begin
+        plugin = Fluent::Plugin::FileBuffer.new
+        plugin.owner = @d
+        rand_num = rand(0..100)
+        bufpath = File.join(File.expand_path("../../tmp/buffer_file_#{rand_num}", __FILE__), 'testbuf.*.log')
+        bufdir = File.dirname(bufpath)
+
+        FileUtils.rm_r bufdir if File.exist?(bufdir)
+        assert !File.exist?(bufdir)
+
+        plugin.configure(config_element('buffer', '', { 'path' => bufpath }))
+
+        assert !File.exist?(bufdir)
+        plugin.start
+
+        m = metadata()
+        c = nil
+        Fluent::SystemConfig.overwrite_system_config("file_permission" => "700") do
+           c = plugin.generate_chunk(m)
+        end
+
+        assert c.is_a? Fluent::Plugin::Buffer::FileChunk
+        assert_equal m, c.metadata
+        assert c.empty?
+        assert_equal :unstaged, c.state
+        assert_equal 0700, c.permission
+        assert_equal bufpath.gsub('.*.', ".b#{Fluent::UniqueId.hex(c.unique_id)}."), c.path
+        assert{ File.stat(c.path).mode.to_s(8).end_with?('700') }
+
+        c.purge
+
+        plugin.stop; plugin.before_shutdown; plugin.shutdown; plugin.after_shutdown; plugin.close; plugin.terminate
+      ensure
+        FileUtils.rm_r bufdir
+      end
+    end
   end
 
   sub_test_case 'configured with system root directory and plugin @id' do

data/test/plugin/test_buf_file_single.rb

@@ -293,6 +293,38 @@ class FileSingleBufferTest < Test::Unit::TestCase
 
       c.purge
     end
+
+    test '#generate_chunk generates blank file chunk with specified permission with system_config' do
+      omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
+
+      @d = create_driver(%[
+        <buffer tag>
+          @type file_single
+          path #{PATH}
+        </buffer>
+      ])
+      @p = @d.instance.buffer
+
+      FileUtils.rm_r @bufdir if File.exist?(@bufdir)
+      assert !File.exist?(@bufdir)
+
+      @p.start
+
+      m = metadata()
+      c = nil
+      Fluent::SystemConfig.overwrite_system_config("file_permission" => "700") do
+        c = @p.generate_chunk(m)
+      end
+      assert c.is_a? Fluent::Plugin::Buffer::FileSingleChunk
+      assert_equal m, c.metadata
+      assert c.empty?
+      assert_equal :unstaged, c.state
+      assert_equal 0700, c.permission
+      assert_equal File.join(@bufdir, "fsb.testing.b#{Fluent::UniqueId.hex(c.unique_id)}.buf"), c.path
+      assert{ File.stat(c.path).mode.to_s(8).end_with?('700') }
+
+      c.purge
+    end
   end
 
   sub_test_case 'configured with system root directory and plugin @id' do

data/test/plugin/test_buffer_file_chunk.rb

@@ -416,17 +416,6 @@ class BufferFileChunkTest < Test::Unit::TestCase
       assert_equal d4.to_json + "\n", lines[3]
     end
 
-    test 'can refer system config for file permission' do
-      omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
-
-      chunk_path = File.join(@chunkdir, 'testperm.*.log')
-      Fluent::SystemConfig.overwrite_system_config("file_permission" => "600") do
-        c = Fluent::Plugin::Buffer::FileChunk.new(gen_metadata, chunk_path, :create)
-        assert{ File.stat(c.path).mode.to_s(8).end_with?('600') }
-        assert{ File.stat(c.path + '.meta').mode.to_s(8).end_with?('600') }
-      end
-    end
-
     test '#write_metadata tries to store metadata on file' do
       d1 = {"f1" => 'v1', "f2" => 'v2', "f3" => 'v3'}
       d2 = {"f1" => 'vv1', "f2" => 'vv2', "f3" => 'vv3'}

data/test/plugin/test_buffer_file_single_chunk.rb

@@ -353,16 +353,6 @@ class BufferFileSingleChunkTest < Test::Unit::TestCase
       assert_equal d3.to_json + "\n", lines[2]
       assert_equal d4.to_json + "\n", lines[3]
     end
-
-    test 'can refer system config for file permission' do
-      omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
-
-      chunk_path = File.join(@chunkdir, 'fsb.*.buf')
-      Fluent::SystemConfig.overwrite_system_config("file_permission" => "600") do
-        c = Fluent::Plugin::Buffer::FileSingleChunk.new(gen_metadata, chunk_path, :create, nil)
-        assert{ File.stat(c.path).mode.to_s(8).end_with?('600') }
-      end
-    end
   end
 
   sub_test_case 'chunk with file for staged chunk' do

data/test/plugin/test_in_monitor_agent.rb

@@ -12,6 +12,8 @@ require_relative '../test_plugin_classes'
 class MonitorAgentInputTest < Test::Unit::TestCase
   include FuzzyAssert
 
+  CONFIG_DIR = File.expand_path('../tmp/in_monitor_agent', __dir__)
+
   def setup
     Fluent::Test.setup
   end
@@ -160,9 +162,20 @@ EOC
     test "fluentd opts" do
       d = create_driver
       opts = Fluent::Supervisor.default_options
-      Fluent::Supervisor.new(opts)
+
+      filepath = nil
+      begin
+        FileUtils.mkdir_p(CONFIG_DIR)
+        filepath = File.expand_path('fluentd.conf', CONFIG_DIR)
+        FileUtils.touch(filepath)
+        s = Fluent::Supervisor.new(opts.merge(config_path: filepath))
+        s.configure
+      ensure
+        FileUtils.rm_r(CONFIG_DIR) rescue _
+      end
+
       expected_opts = {
-        "config_path" => "/etc/fluent/fluent.conf",
+        "config_path" => filepath,
         "pid_file"    => nil,
         "plugin_dirs" => ["/etc/fluent/plugin"],
         "log_path"    => nil,
@@ -279,11 +292,25 @@ EOC
   </match>
 </label>
 EOC
-      @ra = Fluent::RootAgent.new(log: $log)
-      stub(Fluent::Engine).root_agent { @ra }
-      @ra = configure_ra(@ra, conf)
-      # store Supervisor instance to avoid collected by GC
-      @supervisor = Fluent::Supervisor.new(Fluent::Supervisor.default_options)
+
+
+      begin
+        @ra = Fluent::RootAgent.new(log: $log)
+        stub(Fluent::Engine).root_agent { @ra }
+        @ra = configure_ra(@ra, conf)
+        # store Supervisor instance to avoid collected by GC
+
+        FileUtils.mkdir_p(CONFIG_DIR)
+        @filepath = File.expand_path('fluentd.conf', CONFIG_DIR)
+        File.open(@filepath, 'w') do |v|
+          v.puts(conf)
+        end
+
+        @supervisor = Fluent::Supervisor.new(Fluent::Supervisor.default_options.merge(config_path: @filepath))
+        @supervisor.configure
+      ensure
+        FileUtils.rm_r(CONFIG_DIR) rescue _
+      end
     end
 
     test "/api/plugins" do
@@ -469,8 +496,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
   tag monitor
 ")
       d.instance.start
-      expected_response_regex = /pid:\d+\tppid:\d+\tconfig_path:\/etc\/fluent\/fluent.conf\tpid_file:\tplugin_dirs:\/etc\/fluent\/plugin\tlog_path:/
-
+      expected_response_regex = %r{pid:\d+\tppid:\d+\tconfig_path:#{@filepath}\tpid_file:\tplugin_dirs:/etc/fluent/plugin\tlog_path:}
       assert_match(expected_response_regex,
                    get("http://127.0.0.1:#{@port}/api/config").body)
     end
@@ -484,7 +510,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
 ")
       d.instance.start
       res = JSON.parse(get("http://127.0.0.1:#{@port}/api/config.json").body)
-      assert_equal("/etc/fluent/fluent.conf", res["config_path"])
+      assert_equal(@filepath, res["config_path"])
       assert_nil(res["pid_file"])
       assert_equal(["/etc/fluent/plugin"], res["plugin_dirs"])
       assert_nil(res["log_path"])