fluentd 0.14.10-x86-mingw32 → 0.14.11-x86-mingw32

data/test/config/test_system_config.rb

@@ -14,6 +14,7 @@ module Fluent::Config
 
   class FakeSupervisor
     def initialize
+      @root_dir = nil
       @log = FakeLoggerInitializer.new
       @log_level = nil
       @suppress_interval = nil
@@ -27,6 +28,7 @@ module Fluent::Config
   end
 
   class TestSystemConfig < ::Test::Unit::TestCase
+    TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/tmp/system_config/#{ENV['TEST_ENV_NUMBER']}")
 
     def parse_text(text)
       basepath = File.expand_path(File.dirname(__FILE__) + '/../../')
@@ -41,11 +43,13 @@ module Fluent::Config
       s = FakeSupervisor.new
       sc = Fluent::SystemConfig.new(conf)
       sc.apply(s)
+      assert_nil(sc.root_dir)
       assert_nil(sc.log_level)
       assert_nil(sc.suppress_repeated_stacktrace)
       assert_nil(sc.emit_error_log_interval)
       assert_nil(sc.suppress_config_dump)
       assert_nil(sc.without_source)
+      assert_nil(s.instance_variable_get(:@root_dir))
       assert_nil(s.instance_variable_get(:@log_level))
       assert_nil(s.instance_variable_get(:@suppress_repeated_stacktrace))
       assert_nil(s.instance_variable_get(:@emit_error_log_interval))
@@ -55,41 +59,45 @@ module Fluent::Config
       assert_nil(s.instance_variable_get(:@dir_permission))
     end
 
-    {'log_level' => 'error',
-     'suppress_repeated_stacktrace' => true,
-     'emit_error_log_interval' => 60,
-     'suppress_config_dump' => true,
-     'without_source' => true,
-    }.each { |k, v|
-      test "accepts #{k} parameter" do
-        conf = parse_text(<<-EOS)
+    data(
+      'root_dir' => ['root_dir', File.join(TMP_DIR, 'root')],
+      'log_level' => ['log_level', 'error'],
+      'suppress_repeated_stacktrace' => ['suppress_repeated_stacktrace', true],
+      'emit_error_log_interval' => ['emit_error_log_interval', 60],
+      'suppress_config_dump' => ['suppress_config_dump', true],
+      'without_source' => ['without_source', true],
+    )
+    test "accepts parameters" do |(k, v)|
+      conf = parse_text(<<-EOS)
           <system>
             #{k} #{v}
           </system>
-        EOS
-        s = FakeSupervisor.new
-        sc = Fluent::SystemConfig.new(conf)
-        sc.apply(s)
-        assert_not_nil(sc.instance_variable_get("@#{k}"))
-        key = (k == 'emit_error_log_interval' ? 'suppress_interval' : k)
-        assert_not_nil(s.instance_variable_get("@#{key}"))
-      end
-    }
+      EOS
+      s = FakeSupervisor.new
+      sc = Fluent::SystemConfig.new(conf)
+      sc.apply(s)
+      assert_not_nil(sc.instance_variable_get("@#{k}"))
+      key = (k == 'emit_error_log_interval' ? 'suppress_interval' : k)
+      assert_not_nil(s.instance_variable_get("@#{key}"))
+    end
 
-    {'foo' => 'bar', 'hoge' => 'fuga'}.each { |k, v|
-      test "should not affect settable parameters with unknown #{k} parameter" do
-        s = FakeSupervisor.new
-        sc = Fluent::SystemConfig.new({k => v})
-        sc.apply(s)
-        assert_nil(s.instance_variable_get(:@log_level))
-        assert_nil(s.instance_variable_get(:@suppress_repeated_stacktrace))
-        assert_nil(s.instance_variable_get(:@emit_error_log_interval))
-        assert_nil(s.instance_variable_get(:@suppress_config_dump))
-        assert_nil(s.instance_variable_get(:@without_source))
-        assert_nil(s.instance_variable_get(:@file_permission))
-        assert_nil(s.instance_variable_get(:@dir_permission))
-      end
-    }
+    data(
+      'foo' => ['foo', 'bar'],
+      'hoge' => ['hoge', 'fuga'],
+    )
+    test "should not affect settable parameters with unknown parameters" do |(k, v)|
+      s = FakeSupervisor.new
+      sc = Fluent::SystemConfig.new({k => v})
+      sc.apply(s)
+      assert_nil(s.instance_variable_get(:@root_dir))
+      assert_nil(s.instance_variable_get(:@log_level))
+      assert_nil(s.instance_variable_get(:@suppress_repeated_stacktrace))
+      assert_nil(s.instance_variable_get(:@emit_error_log_interval))
+      assert_nil(s.instance_variable_get(:@suppress_config_dump))
+      assert_nil(s.instance_variable_get(:@without_source))
+      assert_nil(s.instance_variable_get(:@file_permission))
+      assert_nil(s.instance_variable_get(:@dir_permission))
+    end
 
     test 'log_level' do
       conf = parse_text(<<-EOS)

data/test/plugin/test_base.rb

@@ -56,6 +56,26 @@ class BaseTest < Test::Unit::TestCase
     assert !@p.has_router?
   end
 
+  sub_test_case '#fluentd_worker_id' do
+    test 'returns 0 in default' do
+      assert_equal 0, @p.fluentd_worker_id
+    end
+
+    test 'returns the value specified via SERVERENGINE_WORKER_ID env variable' do
+      pre_value = ENV['SERVERENGINE_WORKER_ID']
+      begin
+        ENV['SERVERENGINE_WORKER_ID'] = 7.to_s
+        assert_equal 7, @p.fluentd_worker_id
+      ensure
+        ENV['SERVERENGINE_WORKER_ID'] = pre_value
+      end
+    end
+  end
+
+  test 'does not have root dir in default' do
+    assert_nil @p.plugin_root_dir
+  end
+
   test 'is configurable by config_param and config_section' do
     assert_nothing_raised do
       class FluentPluginBaseTest::DummyPlugin2 < Fluent::Plugin::TestBase

data/test/plugin/test_buf_file.rb

@@ -10,6 +10,9 @@ require 'msgpack'
 module FluentPluginFileBufferTest
   class DummyOutputPlugin < Fluent::Plugin::Output
     Fluent::Plugin.register_output('buffer_file_test_output', self)
+    def write(chunk)
+      # drop
+    end
   end
 end
 
@@ -233,6 +236,43 @@ class FileBufferTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case 'configured with system root directory and plugin @id' do
+    setup do
+      @root_dir = File.expand_path('../../tmp/buffer_file_root', __FILE__)
+      FileUtils.rm_rf @root_dir
+
+      Fluent::Test.setup
+      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
+      @p = Fluent::Plugin::FileBuffer.new
+      @p.owner = @d
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => @root_dir) do
+        @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}))
+        @p.configure(config_element('buffer', ''))
+      end
+    end
+
+    teardown do
+      if @p
+        @p.stop unless @p.stopped?
+        @p.before_shutdown unless @p.before_shutdown?
+        @p.shutdown unless @p.shutdown?
+        @p.after_shutdown unless @p.after_shutdown?
+        @p.close unless @p.closed?
+        @p.terminate unless @p.terminated?
+      end
+    end
+
+    test '#start creates directory for buffer chunks' do
+      expected_buffer_path = File.join(@root_dir, 'worker0', 'dummy_output_with_buf', 'buffer', 'buffer.*.log')
+      expected_buffer_dir = File.dirname(expected_buffer_path)
+      assert_equal expected_buffer_path, @p.path
+      assert_false Dir.exist?(expected_buffer_dir)
+
+      @p.start
+
+      assert Dir.exist?(expected_buffer_dir)
+    end
+  end
 
   sub_test_case 'there are no existing file chunks' do
     setup do

data/test/plugin/test_formatter_csv.rb

@@ -27,6 +27,14 @@ class CsvFormatterTest < ::Test::Unit::TestCase
     assert_equal(['a', 'b', 'c'], d.instance.fields)
   end
 
+  data('empty array' => [],
+       'array including empty string' => ['', ''])
+  def test_empty_fields(param)
+    assert_raise Fluent::ConfigError do
+      create_driver('fields' => param)
+    end
+  end
+
   data(
     'tab_char' => ["\t", '\t'],
     'tab_string' => ["\t", 'TAB'],

data/test/plugin/test_in_forward.rb

@@ -34,7 +34,9 @@ class ForwardInputTest < Test::Unit::TestCase
     port #{PORT}
     bind 127.0.0.1
   ]
-  PEERADDR = ['?', '0000', '127.0.0.1', '127.0.0.1']
+  LOCALHOST_HOSTNAME_GETTER = ->(){sock = UDPSocket.new(::Socket::AF_INET); sock.do_not_reverse_lookup = false; sock.connect("127.0.0.1", 2048); sock.peeraddr[2] }
+  LOCALHOST_HOSTNAME = LOCALHOST_HOSTNAME_GETTER.call
+  DUMMY_SOCK = Struct.new(:remote_host, :remote_addr, :remote_port).new(LOCALHOST_HOSTNAME, "127.0.0.1", 0)
   CONFIG_AUTH = %[
     port #{PORT}
     bind 127.0.0.1
@@ -452,7 +454,7 @@ class ForwardInputTest < Test::Unit::TestCase
 
       d.run do
         Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
-          option = d.instance.send(:on_message, obj, chunk.size, PEERADDR)
+          option = d.instance.send(:on_message, obj, chunk.size, DUMMY_SOCK)
           assert_equal 'gzip', option['compressed']
         end
       end
@@ -482,7 +484,7 @@ class ForwardInputTest < Test::Unit::TestCase
 
       d.run do
         Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
-          option = d.instance.send(:on_message, obj, chunk.size, PEERADDR)
+          option = d.instance.send(:on_message, obj, chunk.size, DUMMY_SOCK)
           assert_equal 'gzip', option['compressed']
         end
       end
@@ -506,7 +508,7 @@ class ForwardInputTest < Test::Unit::TestCase
 
       d.run(shutdown: false) do
         Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
-          d.instance.send(:on_message, obj, chunk.size, '127.0.0.1')
+          d.instance.send(:on_message, obj, chunk.size, DUMMY_SOCK)
         end
       end
 
@@ -520,7 +522,7 @@ class ForwardInputTest < Test::Unit::TestCase
       logs = d.instance.log.logs
       assert_equal 1, logs.select{|line|
         line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_warn_limit':/ &&
-        line =~ / tag="test.tag" host="127.0.0.1" limit=16777216 size=16777501/
+        line =~ / tag="test.tag" host="#{LOCALHOST_HOSTNAME}" limit=16777216 size=16777501/
       }.size, "large chunk warning is not logged"
 
       d.instance_shutdown
@@ -538,7 +540,7 @@ class ForwardInputTest < Test::Unit::TestCase
 
       d.run(shutdown: false) do
         Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
-          d.instance.send(:on_message, obj, chunk.size, '127.0.0.1')
+          d.instance.send(:on_message, obj, chunk.size, DUMMY_SOCK)
         end
       end
 
@@ -546,7 +548,7 @@ class ForwardInputTest < Test::Unit::TestCase
       logs = d.instance.log.logs
       assert_equal 1, logs.select{ |line|
         line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_warn_limit':/ &&
-        line =~ / tag="test.tag" host="127.0.0.1" limit=16777216 size=16777501/
+        line =~ / tag="test.tag" host="#{LOCALHOST_HOSTNAME}" limit=16777216 size=16777501/
       }.size, "large chunk warning is not logged"
 
       d.instance_shutdown
@@ -568,7 +570,7 @@ class ForwardInputTest < Test::Unit::TestCase
       # d.run => send_data
       d.run(shutdown: false) do
         Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
-          d.instance.send(:on_message, obj, chunk.size, '127.0.0.1')
+          d.instance.send(:on_message, obj, chunk.size, DUMMY_SOCK)
         end
       end
 
@@ -580,7 +582,7 @@ class ForwardInputTest < Test::Unit::TestCase
       logs = d.instance.log.logs
       assert_equal 1, logs.select{|line|
         line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_limit', dropped:/ &&
-        line =~ / tag="test.tag" host="127.0.0.1" limit=33554432 size=33554989/
+        line =~ / tag="test.tag" host="#{LOCALHOST_HOSTNAME}" limit=33554432 size=33554989/
       }.size, "large chunk warning is not logged"
 
       d.instance_shutdown
@@ -593,7 +595,7 @@ class ForwardInputTest < Test::Unit::TestCase
 
       # d.run => send_data
       d.run(shutdown: false) do
-        d.instance.send(:on_message, data, 1000000000, '127.0.0.1')
+        d.instance.send(:on_message, data, 1000000000, DUMMY_SOCK)
       end
 
       # check emitted data
@@ -602,7 +604,7 @@ class ForwardInputTest < Test::Unit::TestCase
       # check log
       logs = d.instance.log.logs
       assert_equal 1, logs.select{|line|
-        line =~ / \[warn\]: incoming chunk is broken: host="127.0.0.1" msg=#{data.inspect}/
+        line =~ / \[warn\]: incoming chunk is broken: host="#{LOCALHOST_HOSTNAME}" msg=#{data.inspect}/
       }.size, "should not accept broken chunk"
 
       d.instance_shutdown
@@ -1033,17 +1035,27 @@ class ForwardInputTest < Test::Unit::TestCase
     io.close rescue nil # SSL socket requires any writes to close sockets
   end
 
-  sub_test_case 'source_hostname_key feature' do
-    test 'message protocol with source_hostname_key' do
-      execute_test { |events|
+  sub_test_case 'source_hostname_key and source_address_key features' do
+    data(
+      both: [:hostname, :address],
+      hostname: [:hostname],
+      address: [:address],
+    )
+    test 'message protocol' do |keys|
+      execute_test_with_source_hostname_key(*keys) { |events|
         events.each { |tag, time, record|
           send_data [tag, time, record].to_msgpack
         }
       }
     end
 
-    test 'forward protocol with source_hostname_key' do
-      execute_test { |events|
+    data(
+      both: [:hostname, :address],
+      hostname: [:hostname],
+      address: [:address],
+    )
+    test 'forward protocol' do |keys|
+      execute_test_with_source_hostname_key(*keys) { |events|
         entries = []
         events.each {|tag,time,record|
           entries << [time, record]
@@ -1052,8 +1064,13 @@ class ForwardInputTest < Test::Unit::TestCase
       }
     end
 
-    test 'packed forward protocol with source_hostname_key' do
-      execute_test { |events|
+    data(
+      both: [:hostname, :address],
+      hostname: [:hostname],
+      address: [:address],
+    )
+    test 'packed forward protocol' do |keys|
+      execute_test_with_source_hostname_key(*keys) { |events|
         entries = ''
         events.each { |tag, time, record|
           Fluent::Engine.msgpack_factory.packer(entries).write([time, record]).flush
@@ -1063,8 +1080,19 @@ class ForwardInputTest < Test::Unit::TestCase
     end
   end
 
-  def execute_test(&block)
-    @d = d = create_driver(CONFIG + 'source_hostname_key source')
+  def execute_test_with_source_hostname_key(*keys, &block)
+    conf = CONFIG.dup
+    if keys.include?(:hostname)
+      conf << <<EOL
+source_hostname_key source_hostname
+EOL
+    end
+    if keys.include?(:address)
+      conf << <<EOL
+source_address_key source_address
+EOL
+    end
+    @d = d = create_driver(conf)
 
     time = event_time("2011-01-02 13:14:15 UTC")
     events = [
@@ -1077,7 +1105,14 @@ class ForwardInputTest < Test::Unit::TestCase
     end
 
     d.events.each { |tag, _time, record|
-      assert_true record.has_key?('source')
+      if keys.include?(:hostname)
+        assert_true record.has_key?('source_hostname')
+        assert_equal DUMMY_SOCK.remote_host, record['source_hostname']
+      end
+      if keys.include?(:address)
+        assert_true record.has_key?('source_address')
+        assert_equal DUMMY_SOCK.remote_addr, record['source_address']
+      end
     }
   end
 

data/test/plugin/test_in_monitor_agent.rb

@@ -146,7 +146,8 @@ EOC
         "config_path" => "/etc/fluent/fluent.conf",
         "pid_file"    => nil,
         "plugin_dirs" => ["/etc/fluent/plugin"],
-        "log_path"    => nil
+        "log_path"    => nil,
+        "root_dir"    => nil,
       }
       assert_equal(expected_opts, d.instance.fluentd_opts)
     end
@@ -274,16 +275,16 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
       assert_equal(expected_test_filter_response, test_filter)
     end
 
-    data(:include_config_yes => [true, "include_config yes"],
-         :include_config_no => [false, "include_config no"])
-    test "/api/plugins.json" do |(with_config, include_conf)|
-
+    data(:include_config_and_retry_yes => [true, true, "include_config yes", "include_retry yes"],
+         :include_config_and_retry_no => [false, false, "include_config no", "include_retry no"],)
+    test "/api/plugins.json" do |(with_config, with_retry, include_conf, retry_conf)|
       d = create_driver("
   @type monitor_agent
   bind '127.0.0.1'
   port #{@port}
   tag monitor
   #{include_conf}
+  #{retry_conf}
 ")
       d.instance.start
       expected_test_in_response = {
@@ -304,6 +305,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "type"            => "null"
       }
       expected_null_response.merge!("config" => {"@id" => "null", "@type" => "null"}) if with_config
+      expected_null_response.merge!("retry" => {}) if with_retry
       response = JSON.parse(get("http://127.0.0.1:#{@port}/api/plugins.json"))
       test_in_response = response["plugins"][0]
       null_response = response["plugins"][5]
@@ -311,9 +313,9 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
       assert_equal(expected_null_response, null_response)
     end
 
-    data(:with_config_yes => [true, "?with_config=yes"],
-         :with_config_no => [false, "?with_config=no"])
-    test "/api/plugins.json with query parameter. query parameter is preferred than include_config" do |(with_config, query_param)|
+    data(:with_config_and_retry_yes => [true, true, "?with_config=yes&with_retry"],
+         :with_config_and_retry_no => [false, false, "?with_config=no&with_retry=no"])
+    test "/api/plugins.json with query parameter. query parameter is preferred than include_config" do |(with_config, with_retry, query_param)|
 
       d = create_driver("
   @type monitor_agent
@@ -340,6 +342,7 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
         "type"            => "null"
       }
       expected_null_response.merge!("config" => {"@id" => "null", "@type" => "null"}) if with_config
+      expected_null_response.merge!("retry" => {}) if with_retry
       response = JSON.parse(get("http://127.0.0.1:#{@port}/api/plugins.json#{query_param}"))
       test_in_response = response["plugins"][0]
       null_response = response["plugins"][5]
@@ -376,4 +379,73 @@ plugin_id:test_filter\tplugin_category:filter\ttype:test_filter\toutput_plugin:f
       assert_nil(res["log_path"])
     end
   end
+
+  sub_test_case "check retry of buffered plugins" do
+    class FluentTestFailWriteOutput < ::Fluent::Plugin::Output
+      ::Fluent::Plugin.register_output('test_out_fail_write', self)
+
+      def write(chunk)
+        raise "chunk error!"
+      end
+    end
+
+    setup do
+      @port = unused_port
+      # check @type and type in one configuration
+      conf = <<-EOC
+<source>
+  @type monitor_agent
+  @id monitor_agent
+  bind "127.0.0.1"
+  port #{@port}
+</source>
+<match **>
+  @type test_out_fail_write
+  @id test_out_fail_write
+  <buffer>
+    flush_mode immediate
+  </buffer>
+</match>
+      EOC
+      @ra = Fluent::RootAgent.new(log: $log)
+      stub(Fluent::Engine).root_agent { @ra }
+      @ra = configure_ra(@ra, conf)
+    end
+
+    test "/api/plugins.json retry object should be filled if flush was failed" do
+
+      d = create_driver("
+  @type monitor_agent
+  bind '127.0.0.1'
+  port #{@port}
+  include_config no
+")
+      d.instance.start
+      expected_test_out_fail_write_response = {
+          "buffer_queue_length" => 1,
+          "buffer_total_queued_size" => 0,
+          "output_plugin" => true,
+          "plugin_category" => "output",
+          "plugin_id" => "test_out_fail_write",
+          "retry_count" => 2,
+          "type" => "test_out_fail_write",
+          "retry" => {
+              "steps" => 1
+          }
+      }
+      output = @ra.outputs[0]
+      output.start
+      output.after_start
+      output.emit_events('test.tag', Fluent::ArrayEventStream.new([[event_time, {"message" => "test failed flush 1"}]]))
+      # flush few times to check steps
+      2.times do
+        output.force_flush
+      end
+      response = JSON.parse(get("http://127.0.0.1:#{@port}/api/plugins.json"))
+      test_out_fail_write_response = response["plugins"][1]
+      # remove dynamic keys
+      ["start", "next_time"].each { |key| test_out_fail_write_response["retry"].delete(key) }
+      assert_equal(expected_test_out_fail_write_response, test_out_fail_write_response)
+    end
+  end
 end