fluentd 0.14.6 → 0.14.7

data/test/plugin/test_in_monitor_agent.rb

@@ -55,6 +55,19 @@ class MonitorAgentInputTest < Test::Unit::TestCase
     @id test_out
   </match>
 </label>
+<label @copy>
+  <match **>
+    @type copy
+    <store>
+      @type test_out
+      @id copy_out_1
+    </store>
+    <store>
+      @type test_out
+      @id copy_out_2
+    </store>
+  </match>
+</label>
 <label @ERROR>
   <match>
     @type null
@@ -153,7 +166,10 @@ EOC
       assert_equal([FluentTest::FluentTestInput,
                     Fluent::Plugin::RelabelOutput,
                     FluentTest::FluentTestFilter,
-                    FluentTest::FluentTestOutput,
+                    FluentTest::FluentTestOutput, # in label @test
+                    Fluent::Plugin::CopyOutput,
+                    FluentTest::FluentTestOutput, # in label @copy 1
+                    FluentTest::FluentTestOutput, # in label @copy 2
                     Fluent::Plugin::NullOutput], plugins)
     end
 

data/test/plugin/test_in_tail.rb

@@ -5,6 +5,7 @@ require 'fluent/plugin/buffer'
 require 'fluent/system_config'
 require 'net/http'
 require 'flexmock/test_unit'
+require 'timecop'
 
 class TailInputTest < Test::Unit::TestCase
   include FlexMock::TestCase
@@ -401,7 +402,7 @@ class TailInputTest < Test::Unit::TestCase
 
     d = create_driver
 
-    d.run do
+    d.run(expect_emits: 1) do
       File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
         f.print "test3"
       }
@@ -410,7 +411,6 @@ class TailInputTest < Test::Unit::TestCase
       File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
         f.puts "test4"
       }
-      sleep 1
     end
 
     events = d.events
@@ -766,9 +766,7 @@ class TailInputTest < Test::Unit::TestCase
       @loop = Coolio::Loop.new
     end
 
-    flexstub(Time) do |timeclass|
-      timeclass.should_receive(:now).with_no_args.and_return(Time.new(2010, 1, 2, 3, 4, 5), Time.new(2010, 1, 2, 3, 4, 6), Time.new(2010, 1, 2, 3, 4, 7))
-
+    Timecop.freeze(2010, 1, 2, 3, 4, 5) do
       flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
         EX_PATHS.each do |path|
           watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, 1000, any, any, any).once.and_return do
@@ -781,11 +779,13 @@ class TailInputTest < Test::Unit::TestCase
         end
         plugin.refresh_watchers
       end
+    end
 
-      plugin.instance_eval do
-        @tails['test/plugin/data/2010/01/20100102-030405.log'].should_receive(:close).zero_or_more_times
-      end
+    plugin.instance_eval do
+      @tails['test/plugin/data/2010/01/20100102-030405.log'].should_receive(:close).zero_or_more_times
+    end
 
+    Timecop.freeze(2010, 1, 2, 3, 4, 6) do
       flexstub(Fluent::Plugin::TailInput::TailWatcher) do |watcherclass|
         watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::Plugin::TailInput::FilePositionEntry, any, true, true, 1000, any, any, any).once.and_return do
           flexmock('TailWatcher') do |watcher|

data/test/plugin/test_out_file.rb

@@ -24,14 +24,6 @@ class FileOutputTest < Test::Unit::TestCase
     Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::FileOutput).configure(conf)
   end
 
-  def with_timezone(timezone = 'UTC', &block)
-    old = ENV['TZ']
-    ENV['TZ'] = timezone
-    output = yield
-    ENV['TZ'] = old
-    output
-  end
-
   def test_configure
     d = create_driver %[
       path test_path

data/test/plugin/test_out_forward.rb

@@ -2,6 +2,7 @@ require_relative '../helper'
 require 'fluent/test'
 require 'fluent/test/startup_shutdown'
 require 'fluent/plugin/out_forward'
+require 'flexmock/test_unit'
 
 class ForwardOutputTest < Test::Unit::TestCase
   extend Fluent::Test::StartupShutdown
@@ -107,6 +108,40 @@ class ForwardOutputTest < Test::Unit::TestCase
     end
   end
 
+  def test_compress_default_value
+    d = create_driver
+    assert_equal :text, d.instance.compress
+
+    node = d.instance.nodes.first
+    assert_equal :text, node.instance_variable_get(:@compress)
+  end
+
+  def test_set_compress_is_gzip
+    d = create_driver(CONFIG + %[compress gzip])
+    assert_equal :gzip, d.instance.compress
+    assert_equal :gzip, d.instance.buffer.compress
+
+    node = d.instance.nodes.first
+    assert_equal :gzip, node.instance_variable_get(:@compress)
+  end
+
+  def test_set_compress_is_gzip_in_buffer_section
+    mock = flexmock($log)
+    mock.should_receive(:log).with("buffer is compressed.  If you also want to save the bandwidth of a network, Add `compress` configuration in <match>")
+
+    d = create_driver(CONFIG + %[
+       <buffer>
+         type memory
+         compress gzip
+       </buffer>
+     ])
+    assert_equal :text, d.instance.compress
+    assert_equal :gzip, d.instance.buffer.compress
+
+    node = d.instance.nodes.first
+    assert_equal :text, node.instance_variable_get(:@compress)
+  end
+
   def test_phi_failure_detector
     d = create_driver(CONFIG + %[phi_failure_detector false \n phi_threshold 0])
     node = d.instance.nodes.first
@@ -202,6 +237,40 @@ class ForwardOutputTest < Test::Unit::TestCase
     assert_empty d.instance.exceptions
   end
 
+  def test_send_comprssed_message_pack_stream_if_compress_is_gzip
+    target_input_driver = create_target_input_driver
+
+    d = create_driver(CONFIG + %[
+      flush_interval 1s
+      compress gzip
+    ])
+
+    time = event_time('2011-01-02 13:14:15 UTC')
+
+    records = [
+      {"a" => 1},
+      {"a" => 2}
+    ]
+    d.register_run_post_condition do
+      d.instance.responses.length == 1
+    end
+
+    target_input_driver.run do
+      d.run do
+        records.each do |record|
+          d.emit record, time
+        end
+      end
+    end
+
+    event_streams = target_input_driver.event_streams
+    assert_true event_streams[0].is_a?(Fluent::CompressedMessagePackEventStream)
+
+    emits = target_input_driver.emits
+    assert_equal ['test', time, records[0]], emits[0]
+    assert_equal ['test', time, records[1]], emits[1]
+  end
+
   def test_send_to_a_node_supporting_responses
     target_input_driver = create_target_input_driver(response_stub: true)
 
@@ -529,4 +598,20 @@ class ForwardOutputTest < Test::Unit::TestCase
     node.tick
     assert_equal node.available, true
   end
+
+  def test_heartbeat_type_udp
+    d = create_driver(CONFIG + "\nheartbeat_type udp")
+
+    d.instance.start
+    usock = d.instance.instance_variable_get(:@usock)
+    timer = d.instance.instance_variable_get(:@timer)
+    hb = d.instance.instance_variable_get(:@hb)
+    assert_equal UDPSocket, usock.class
+    assert_equal Fluent::ForwardOutput::HeartbeatRequestTimer, timer.class
+    assert_equal Fluent::ForwardOutput::HeartbeatHandler, hb.class
+
+    mock(usock).send("\0", 0, Socket.pack_sockaddr_in(TARGET_PORT, '127.0.0.1')).once
+    timer.disable # call send_heartbeat at just once
+    timer.on_timer
+  end
 end

data/test/plugin/test_out_secondary_file.rb

@@ -110,16 +110,19 @@ class FileOutputSecondaryTest < Test::Unit::TestCase
     # Zlib::GzipReader has a bug of concatenated file: https://bugs.ruby-lang.org/issues/9790
     # Following code from https://www.ruby-forum.com/topic/971591#979520
     result = ""
-    File.open(path, "rb") { |io|
-      loop do
-        gzr = Zlib::GzipReader.new(io)
-        result << gzr.read
-        unused = gzr.unused
-        gzr.finish
-        break if unused.nil?
-        io.pos -= unused.length
-      end
-    }
+    waiting(10) do
+      # we can expect that GzipReader#read can wait unflushed raw data of `io` on disk
+      File.open(path, "rb") { |io|
+        loop do
+          gzr = Zlib::GzipReader.new(io)
+          result << gzr.read
+          unused = gzr.unused
+          gzr.finish
+          break if unused.nil?
+          io.pos -= unused.length
+        end
+      }
+    end
 
     assert_equal expect, result
   end
@@ -155,10 +158,15 @@ class FileOutputSecondaryTest < Test::Unit::TestCase
         basename out_file_test
       ], @primary)
 
-      path = d.instance.write(@chunk)
+      msgpack_binary = @es.to_msgpack_stream.force_encoding('ASCII-8BIT')
 
+      path = d.instance.write(@chunk)
       assert_equal "#{TMP_DIR}/out_file_test.0", path
-      assert_equal File.read(path), @es.to_msgpack_stream.force_encoding('ASCII-8BIT')
+      waiting(5) do
+        sleep 0.1 until File.stat(path).size == msgpack_binary.size
+      end
+
+      assert_equal File.read(path), msgpack_binary
     end
 
     test 'path should be incremental when append option is false' do

data/test/plugin/test_out_stdout.rb

@@ -9,6 +9,7 @@ class StdoutOutputTest < Test::Unit::TestCase
 
   CONFIG = %[
   ]
+  TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%9N %z'
 
   def create_driver(conf = CONFIG)
     Fluent::Test::Driver::Output.new(Fluent::Plugin::StdoutOutput).configure(conf)
@@ -40,7 +41,7 @@ class StdoutOutputTest < Test::Unit::TestCase
           d.feed(time, {'test' => 'test1'})
         end
       end
-      assert_equal "#{Time.at(time).localtime} test: {\"test\":\"test1\"}\n", out
+      assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\":\"test1\"}\n", out
     end
 
     data('oj' => 'oj', 'yajl' => 'yajl')
@@ -52,14 +53,14 @@ class StdoutOutputTest < Test::Unit::TestCase
           d.feed(time, {'test' => 'test1'})
         end
       end
-      assert_equal "#{Time.at(time).localtime} test: {\"test\":\"test1\"}\n", out
+      assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\":\"test1\"}\n", out
 
       if data == 'yajl'
         # NOTE: Float::NAN is not jsonable
         assert_raise(Yajl::EncodeError) { d.feed('test', time, {'test' => Float::NAN}) }
       else
         out = capture_log { d.feed('test', time, {'test' => Float::NAN}) }
-        assert_equal "#{Time.at(time).localtime} test: {\"test\":NaN}\n", out
+        assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\":NaN}\n", out
       end
     end
 
@@ -71,11 +72,11 @@ class StdoutOutputTest < Test::Unit::TestCase
           d.feed(time, {'test' => 'test2'})
         end
       end
-      assert_equal "#{Time.at(time).localtime} test: {\"test\"=>\"test2\"}\n", out
+      assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\"=>\"test2\"}\n", out
 
       # NOTE: Float::NAN is not jsonable, but hash string can output it.
       out = capture_log { d.feed('test', time, {'test' => Float::NAN}) }
-      assert_equal "#{Time.at(time).localtime} test: {\"test\"=>NaN}\n", out
+      assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\"=>NaN}\n", out
     end
   end
 
@@ -113,7 +114,7 @@ class StdoutOutputTest < Test::Unit::TestCase
             d.feed(time, {'test' => 'test'})
           end
         end
-        assert_equal "#{Time.at(time).localtime} test: {\"test\":\"test\"}\n", out
+        assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\":\"test\"}\n", out
       end
     end
 
@@ -128,7 +129,7 @@ class StdoutOutputTest < Test::Unit::TestCase
             d.feed(time, {'test' => 'test'})
           end
         end
-        assert_equal "#{Time.at(time).localtime} test: {\"test\":\"test\"}\n", out
+        assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\":\"test\"}\n", out
       end
 
       data('oj' => 'oj', 'yajl' => 'yajl')
@@ -143,7 +144,7 @@ class StdoutOutputTest < Test::Unit::TestCase
           end
         end
 
-        assert_equal "#{Time.at(time).localtime} test: {\"test\":\"test\"}\n", out
+        assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\":\"test\"}\n", out
       end
     end
 
@@ -158,7 +159,7 @@ class StdoutOutputTest < Test::Unit::TestCase
           end
         end
 
-        assert_equal "#{Time.at(time).localtime} test: {\"test\"=>\"test\"}\n", out
+        assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\"=>\"test\"}\n", out
       end
 
       test '#try_write(asynchronous)' do
@@ -172,7 +173,7 @@ class StdoutOutputTest < Test::Unit::TestCase
           end
         end
 
-        assert_equal "#{Time.at(time).localtime} test: {\"test\"=>\"test\"}\n", out
+        assert_equal "#{Time.at(time).localtime.strftime(TIME_FORMAT)} test: {\"test\"=>\"test\"}\n", out
       end
     end
   end

data/test/plugin/test_output.rb

@@ -287,6 +287,208 @@ class OutputTest < Test::Unit::TestCase
       assert_equal "/mypath/2016/04/11/20-30/fluentd.test.output/////tail", @i.extract_placeholders(tmpl, m)
     end
 
+    sub_test_case '#placeholder_validators' do
+      test 'returns validators for time, tag and keys when a template has placeholders even if plugin is not configured with these keys' do
+        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
+        validators = @i.placeholder_validators(:path, "/my/path/${tag}/${username}/file.%Y%m%d_%H%M.log")
+        assert_equal 3, validators.size
+        assert_equal 1, validators.select(&:time?).size
+        assert_equal 1, validators.select(&:tag?).size
+        assert_equal 1, validators.select(&:keys?).size
+      end
+
+      test 'returns validators for time, tag and keys when a plugin is configured with these keys even if a template does not have placeholders' do
+        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,username', {'timekey' => 60})]))
+        validators = @i.placeholder_validators(:path, "/my/path/file.log")
+        assert_equal 3, validators.size
+        assert_equal 1, validators.select(&:time?).size
+        assert_equal 1, validators.select(&:tag?).size
+        assert_equal 1, validators.select(&:keys?).size
+      end
+
+      test 'returns a validator for time if a template has timestamp placeholders' do
+        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
+        validators = @i.placeholder_validators(:path, "/my/path/file.%Y-%m-%d.log")
+        assert_equal 1, validators.size
+        assert_equal 1, validators.select(&:time?).size
+        assert_raise Fluent::ConfigError.new("Parameter 'path' has timestamp placeholders, but chunk key 'time' is not configured") do
+          validators.first.validate!
+        end
+      end
+
+      test 'returns a validator for time if a plugin is configured with time key' do
+        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {'timekey' => '30'})]))
+        validators = @i.placeholder_validators(:path, "/my/path/to/file.log")
+        assert_equal 1, validators.size
+        assert_equal 1, validators.select(&:time?).size
+        assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have timestamp placeholders for timekey 30") do
+          validators.first.validate!
+        end
+      end
+
+      test 'returns a validator for tag if a template has tag placeholders' do
+        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
+        validators = @i.placeholder_validators(:path, "/my/path/${tag}/file.log")
+        assert_equal 1, validators.size
+        assert_equal 1, validators.select(&:tag?).size
+        assert_raise Fluent::ConfigError.new("Parameter 'path' has tag placeholders, but chunk key 'tag' is not configured") do
+          validators.first.validate!
+        end
+      end
+
+      test 'returns a validator for tag if a plugin is configured with tag key' do
+        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'tag')]))
+        validators = @i.placeholder_validators(:path, "/my/path/file.log")
+        assert_equal 1, validators.size
+        assert_equal 1, validators.select(&:tag?).size
+        assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have tag placeholder") do
+          validators.first.validate!
+        end
+      end
+
+      test 'returns a validator for variable keys if a template has variable placeholders' do
+        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
+        validators = @i.placeholder_validators(:path, "/my/path/${username}/file.${group}.log")
+        assert_equal 1, validators.size
+        assert_equal 1, validators.select(&:keys?).size
+        assert_raise Fluent::ConfigError.new("Parameter 'path' has placeholders, but chunk keys doesn't have keys group,username") do
+          validators.first.validate!
+        end
+      end
+
+      test 'returns a validator for variable keys if a plugin is configured with variable keys' do
+        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'username,group')]))
+        validators = @i.placeholder_validators(:path, "/my/path/file.log")
+        assert_equal 1, validators.size
+        assert_equal 1, validators.select(&:keys?).size
+        assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have enough placeholders for keys group,username") do
+          validators.first.validate!
+        end
+      end
+    end
+
+    sub_test_case '#placeholder_validate!' do
+      test 'raises configuration error for a templace when timestamp placeholders exist but time key is missing' do
+        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
+        assert_raise Fluent::ConfigError.new("Parameter 'path' has timestamp placeholders, but chunk key 'time' is not configured") do
+          @i.placeholder_validate!(:path, "/path/without/timestamp/file.%Y%m%d-%H%M.log")
+        end
+      end
+
+      test 'raises configuration error for a template without timestamp placeholders when timekey is configured' do
+        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {"timekey" => 180})]))
+        assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have timestamp placeholders for timekey 180") do
+          @i.placeholder_validate!(:path, "/my/path/file.log")
+        end
+        assert_nothing_raised do
+          @i.placeholder_validate!(:path, "/my/path/%Y%m%d/file.%H%M.log")
+        end
+      end
+
+      test 'raises configuration error for a template with timestamp placeholders when plugin is configured more fine timekey' do
+        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {"timekey" => 180})]))
+        assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have timestamp placeholder for hour('%H') for timekey 180") do
+          @i.placeholder_validate!(:path, "/my/path/file.%Y%m%d_%H.log")
+        end
+        assert_nothing_raised do
+          @i.placeholder_validate!(:path, "/my/path/file.%Y%m%d_%H%M.log")
+        end
+      end
+
+      test 'raises configuration error for a template when tag placeholders exist but tag key is missing' do
+        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
+        assert_raise Fluent::ConfigError.new("Parameter 'path' has tag placeholders, but chunk key 'tag' is not configured") do
+          @i.placeholder_validate!(:path, "/my/path/${tag}/file.${tag[2]}.log")
+        end
+      end
+
+      test 'raises configuration error for a template without tag placeholders when tagkey is configured' do
+        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'tag')]))
+        assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have tag placeholder") do
+          @i.placeholder_validate!(:path, "/my/path/file.log")
+        end
+        assert_nothing_raised do
+          @i.placeholder_validate!(:path, "/my/path/${tag}/file.${tag[2]}.log")
+        end
+      end
+
+      test 'raises configuration error for a template when variable key placeholders exist but chunk keys are missing' do
+        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
+        assert_raise Fluent::ConfigError.new("Parameter 'path' has placeholders, but chunk keys doesn't have keys service,username") do
+          @i.placeholder_validate!(:path, "/my/path/${service}/file.${username}.log")
+        end
+      end
+
+      test 'raises configuration error for a template without variable key placeholders when chunk keys are configured' do
+        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'username,service')]))
+        assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have enough placeholders for keys service,username") do
+          @i.placeholder_validate!(:path, "/my/path/file.log")
+        end
+        assert_nothing_raised do
+          @i.placeholder_validate!(:path, "/my/path/${service}/file.${username}.log")
+        end
+      end
+
+      test 'raise configuration error for a template and configuration with keys mismatch' do
+        @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'username,service')]))
+        assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have enough placeholders for keys service") do
+          @i.placeholder_validate!(:path, "/my/path/file.${username}.log")
+        end
+        assert_raise Fluent::ConfigError.new("Parameter 'path' doesn't have enough placeholders for keys username") do
+          @i.placeholder_validate!(:path, "/my/path/${service}/file.log")
+        end
+        assert_nothing_raised do
+          @i.placeholder_validate!(:path, "/my/path/${service}/file.${username}.log")
+        end
+      end
+    end
+
+    test '#get_placeholders_time returns seconds,title and example placeholder for a template' do
+      s, t, e = @i.get_placeholders_time("/path/to/dir/yay")
+      assert_nil s
+      assert_nil t
+      assert_nil e
+
+      s, t, e = @i.get_placeholders_time("/path/to/%Y%m%d/yay")
+      assert_equal 86400, s
+      assert_equal :day, t
+      assert_equal '%d', e
+      s, t, e = @i.get_placeholders_time("my birthiday! at %F")
+      assert_equal 86400, s
+      assert_equal :day, t
+      assert_equal '%d', e
+
+      s, t, e = @i.get_placeholders_time("myfile.%Y-%m-%d_%H.log")
+      assert_equal 3600, s
+      assert_equal :hour, t
+      assert_equal '%H', e
+
+      s, t, e = @i.get_placeholders_time("part-%Y%m%d-%H%M.ts")
+      assert_equal 60, s
+      assert_equal :minute, t
+      assert_equal '%M', e
+
+      s, t, e = @i.get_placeholders_time("my first data at %F %T %z")
+      assert_equal 1, s
+      assert_equal :second, t
+      assert_equal '%S', e
+    end
+
+    test '#get_placeholders_tag returns a list of tag part position for a template' do
+      assert_equal [], @i.get_placeholders_tag("db.table")
+      assert_equal [], @i.get_placeholders_tag("db.table_${non_tag}")
+      assert_equal [-1], @i.get_placeholders_tag("table_${tag}")
+      assert_equal [0, 1], @i.get_placeholders_tag("db_${tag[0]}.table_${tag[1]}")
+      assert_equal [-1, 0], @i.get_placeholders_tag("/treedir/${tag[0]}/${tag}")
+    end
+
+    test '#get_placeholders_keys returns a list of keys for a template' do
+      assert_equal [], @i.get_placeholders_keys("/path/to/my/data/file.log")
+      assert_equal [], @i.get_placeholders_keys("/path/to/my/${tag}/file.log")
+      assert_equal ['key1', 'key2'], @i.get_placeholders_keys("/path/to/${key2}/${tag}/file.${key1}.log")
+      assert_equal ['.hidden', '0001', '@timestamp', 'a_key', 'my-domain'], @i.get_placeholders_keys("http://${my-domain}/${.hidden}/${0001}/${a_key}?timestamp=${@timestamp}")
+    end
+
     test '#metadata returns object which contains tag/timekey/variables from records as specified in configuration' do
       tag = 'test.output'
       time = event_time('2016-04-12 15:31:23 -0700')
@@ -565,4 +767,36 @@ class OutputTest < Test::Unit::TestCase
       @i.stop; @i.before_shutdown; @i.shutdown; @i.after_shutdown; @i.close; @i.terminate
     end
   end
+
+  sub_test_case '#generate_format_proc' do
+    test "when output doesn't have <buffer>" do
+      i = create_output(:sync)
+      i.configure(config_element('ROOT', '', {}, []))
+      assert_equal Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM, i.generate_format_proc
+    end
+
+    test "when output doesn't have <buffer> and time_as_integer is true" do
+      i = create_output(:sync)
+      i.configure(config_element('ROOT', '', {'time_as_integer' => true}))
+      assert_equal Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM_TIME_INT, i.generate_format_proc
+    end
+
+    test 'when output has <buffer> and compress is gzip' do
+      i = create_output(:buffered)
+      i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {'compress' => 'gzip'})]))
+      assert_equal Fluent::Plugin::Output::FORMAT_COMPRESSED_MSGPACK_STREAM, i.generate_format_proc
+    end
+
+    test 'when output has <buffer> and compress is gzip and time_as_integer is true' do
+      i = create_output(:buffered)
+      i.configure(config_element('ROOT', '', {'time_as_integer' => true}, [config_element('buffer', '', {'compress' => 'gzip'})]))
+      assert_equal Fluent::Plugin::Output::FORMAT_COMPRESSED_MSGPACK_STREAM_TIME_INT, i.generate_format_proc
+    end
+
+    test 'when output has <buffer> and compress is text' do
+      i = create_output(:buffered)
+      i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {'compress' => 'text'})]))
+      assert_equal Fluent::Plugin::Output::FORMAT_MSGPACK_STREAM, i.generate_format_proc
+    end
+  end
 end