fluentd 0.14.17-x64-mingw32 → 1.3.1-x64-mingw32

data/test/plugin/test_out_copy.rb

@@ -156,5 +156,36 @@ class CopyOutputTest < Test::Unit::TestCase
       end
     end
   end
+
+  IGNORE_ERROR_CONFIG = %[
+    <store ignore_error>
+      @type test
+      name c0
+    </store>
+    <store ignore_error>
+      @type test
+      name c1
+    </store>
+    <store>
+      @type test
+      name c2
+    </store>
+  ]
+
+  def test_ignore_error
+    d = create_driver(IGNORE_ERROR_CONFIG)
+
+    # override to raise an error
+    d.instance.outputs[0].define_singleton_method(:process) do |tag, es|
+      raise ArgumentError, 'Failed'
+    end
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    assert_nothing_raised do
+      d.run(default_tag: 'test') do
+        d.feed(time, {"a"=>1})
+      end
+    end
+  end
 end
 

data/test/plugin/test_out_file.rb

@@ -304,7 +304,11 @@ class FileOutputTest < Test::Unit::TestCase
       assert_equal r4, d.formatted[3]
       assert_equal r5, d.formatted[4]
 
-      read_gunzip = ->(path){ File.open(path){|fio| Zlib::GzipReader.open(fio){|io| io.read } } }
+      read_gunzip = ->(path){
+        File.open(path){ |fio|
+          Zlib::GzipReader.new(StringIO.new(fio.read)).read
+        }
+      }
       assert_equal r1 + r2, read_gunzip.call("#{TMP_DIR}/my.data/a/full.20161003.2345.log.gz")
       assert_equal r3, read_gunzip.call("#{TMP_DIR}/your.data/a/full.20161003.2345.log.gz")
       assert_equal r4, read_gunzip.call("#{TMP_DIR}/my.data/a/full.20161004.0000.log.gz")
@@ -370,7 +374,7 @@ class FileOutputTest < Test::Unit::TestCase
     result = ''
     File.open(path, "rb") { |io|
       loop do
-        gzr = Zlib::GzipReader.new(io)
+        gzr = Zlib::GzipReader.new(StringIO.new(io.read))
         result << gzr.read
         unused = gzr.unused
         gzr.finish
@@ -557,6 +561,108 @@ class FileOutputTest < Test::Unit::TestCase
     check_gzipped_result(path, formatted_lines * 3)
   end
 
+  test 'append when JST' do
+    with_timezone(Fluent.windows? ? "JST-9" : "Asia/Tokyo") do
+      time = event_time("2011-01-02 03:14:15+09:00")
+      formatted_lines = %[2011-01-02T03:14:15+09:00\ttest\t{"a":1}\n] + %[2011-01-02T03:14:15+09:00\ttest\t{"a":2}\n]
+
+      write_once = ->(){
+        d = create_driver %[
+          path #{TMP_DIR}/out_file_test
+          compress gz
+          append true
+          <buffer>
+            timekey_use_utc false
+            timekey_zone Asia/Tokyo
+          </buffer>
+        ]
+        d.run(default_tag: 'test'){
+          d.feed(time, {"a"=>1})
+          d.feed(time, {"a"=>2})
+        }
+        d.instance.last_written_path
+      }
+
+      path = write_once.call
+      assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
+      check_gzipped_result(path, formatted_lines)
+
+      path = write_once.call
+      assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
+      check_gzipped_result(path, formatted_lines * 2)
+
+      path = write_once.call
+      assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
+      check_gzipped_result(path, formatted_lines * 3)
+    end
+  end
+
+  test 'append when UTC-02 but timekey_zone is +0900' do
+    with_timezone("UTC-02") do # +0200
+      time = event_time("2011-01-02 17:14:15+02:00")
+      formatted_lines = %[2011-01-02T17:14:15+02:00\ttest\t{"a":1}\n] + %[2011-01-02T17:14:15+02:00\ttest\t{"a":2}\n]
+
+      write_once = ->(){
+        d = create_driver %[
+          path #{TMP_DIR}/out_file_test
+          compress gz
+          append true
+          <buffer>
+            timekey_use_utc false
+            timekey_zone +0900
+          </buffer>
+        ]
+        d.run(default_tag: 'test'){
+          d.feed(time, {"a"=>1})
+          d.feed(time, {"a"=>2})
+        }
+        d.instance.last_written_path
+      }
+
+      path = write_once.call
+      # Rotated at 2011-01-02 17:00:00+02:00
+      assert_equal "#{TMP_DIR}/out_file_test.20110103.log.gz", path
+      check_gzipped_result(path, formatted_lines)
+
+      path = write_once.call
+      assert_equal "#{TMP_DIR}/out_file_test.20110103.log.gz", path
+      check_gzipped_result(path, formatted_lines * 2)
+
+      path = write_once.call
+      assert_equal "#{TMP_DIR}/out_file_test.20110103.log.gz", path
+      check_gzipped_result(path, formatted_lines * 3)
+    end
+  end
+
+  test '${chunk_id}' do
+    time = event_time("2011-01-02 13:14:15 UTC")
+    formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
+
+    write_once = ->(){
+      d = create_driver %[
+        path #{TMP_DIR}/out_file_chunk_id_${chunk_id}
+        utc
+        append true
+        <buffer>
+          timekey_use_utc true
+        </buffer>
+      ]
+      d.run(default_tag: 'test'){
+        d.feed(time, {"a"=>1})
+        d.feed(time, {"a"=>2})
+      }
+      d.instance.last_written_path
+    }
+
+    path = write_once.call
+    if File.basename(path) =~ /out_file_chunk_id_([-_.@a-zA-Z0-9].*).20110102.log/
+      unique_id = Fluent::UniqueId.hex(Fluent::UniqueId.generate)
+      assert_equal unique_id.size, $1.size, "chunk_id size is mismatched"
+    else
+      flunk "chunk_id is not included in the path"
+    end
+  end
+
   test 'symlink' do
     omit "Windows doesn't support symlink" if Fluent.windows?
     conf = CONFIG + %[

data/test/plugin/test_out_forward.rb

@@ -9,6 +9,8 @@ require 'fluent/plugin/in_forward'
 class ForwardOutputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
+    FileUtils.rm_rf(TMP_DIR)
+    FileUtils.mkdir_p(TMP_DIR)
     @d = nil
   end
 
@@ -16,6 +18,8 @@ class ForwardOutputTest < Test::Unit::TestCase
     @d.instance_shutdown if @d
   end
 
+  TMP_DIR = File.join(__dir__, "../tmp/out_forward#{ENV['TEST_ENV_NUMBER']}")
+
   TARGET_HOST = '127.0.0.1'
   TARGET_PORT = unused_port
   CONFIG = %[
@@ -153,6 +157,27 @@ EOL
     assert{ logs.any?{|log| log.include?(expected_log) && log.include?(expected_detail) } }
   end
 
+  data('CA cert'     => 'tls_ca_cert_path',
+       'non CA cert' => 'tls_cert_path')
+  test 'configure tls_cert_path/tls_ca_cert_path' do |param|
+    dummy_cert_path = File.join(TMP_DIR, "dummy_cert.pem")
+    FileUtils.touch(dummy_cert_path)
+    conf = %[
+      send_timeout 5
+      transport tls
+      tls_insecure_mode true
+      #{param} #{dummy_cert_path}
+      <server>
+        host #{TARGET_HOST}
+        port #{TARGET_PORT}
+      </server>
+    ]
+
+    @d = d = create_driver(conf)
+    # In the plugin, tls_ca_cert_path is used for both cases
+    assert_equal([dummy_cert_path], d.instance.tls_ca_cert_path)
+  end
+
   test 'compress_default_value' do
     @d = d = create_driver
     assert_equal :text, d.instance.compress
@@ -217,6 +242,18 @@ EOL
     assert_equal 2, d.instance.ack_response_timeout
   end
 
+  test 'verify_connection_at_startup is disabled in default' do
+    @d = d = create_driver(CONFIG)
+    assert_false d.instance.verify_connection_at_startup
+  end
+
+  test 'verify_connection_at_startup can be enabled' do
+    @d = d = create_driver(CONFIG + %[
+      verify_connection_at_startup true
+    ])
+    assert_true d.instance.verify_connection_at_startup
+  end
+
   test 'send tags in str (utf-8 strings)' do
     target_input_driver = create_target_input_driver
 
@@ -486,6 +523,7 @@ EOL
         retry_type periodic
         retry_wait 30s
         flush_at_shutdown false # suppress errors in d.instance_shutdown
+        flush_thread_interval 30s
       </buffer>
     ])
 
@@ -501,7 +539,7 @@ EOL
     target_input_driver.end_if{ d.instance.rollback_count > 0 }
     target_input_driver.end_if{ !node.available }
     target_input_driver.run(expect_records: 2, timeout: 25) do
-      d.run(default_tag: 'test', timeout: 20, wait_flush_completion: false, shutdown: false) do
+      d.run(default_tag: 'test', timeout: 20, wait_flush_completion: false, shutdown: false, flush: false) do
         delayed_commit_timeout_value = d.instance.delayed_commit_timeout
         d.feed([[time, records[0]], [time,records[1]]])
       end
@@ -530,6 +568,7 @@ EOL
         retry_type periodic
         retry_wait 30s
         flush_at_shutdown false # suppress errors in d.instance_shutdown
+        flush_thread_interval 30s
       </buffer>
     ])
 
@@ -545,7 +584,7 @@ EOL
     target_input_driver.end_if{ d.instance.rollback_count > 0 }
     target_input_driver.end_if{ !node.available }
     target_input_driver.run(expect_records: 2, timeout: 25) do
-      d.run(default_tag: 'test', timeout: 20, wait_flush_completion: false, shutdown: false) do
+      d.run(default_tag: 'test', timeout: 20, wait_flush_completion: false, shutdown: false, flush: false) do
         delayed_commit_timeout_value = d.instance.delayed_commit_timeout
         d.feed([[time, records[0]], [time,records[1]]])
       end
@@ -664,6 +703,51 @@ EOL
     assert_equal(['test', time, records[1]], events[1])
   end
 
+  # This test is not 100% but test failed with previous Node implementation which has race condition
+  test 'Node with security is thread-safe on multi threads' do
+    input_conf = TARGET_CONFIG + %[
+                   <security>
+                     self_hostname in.localhost
+                     shared_key fluentd-sharedkey
+                     <client>
+                       host 127.0.0.1
+                     </client>
+                   </security>
+                 ]
+    target_input_driver = create_target_input_driver(conf: input_conf)
+    output_conf = %[
+      send_timeout 51
+      <security>
+        self_hostname localhost
+        shared_key fluentd-sharedkey
+      </security>
+      <server>
+        name test
+        host #{TARGET_HOST}
+        port #{TARGET_PORT}
+        shared_key fluentd-sharedkey
+      </server>
+    ]
+    @d = d = create_driver(output_conf)
+
+    chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
+    target_input_driver.run(timeout: 15) do
+      d.run(shutdown: false) do
+        node = d.instance.nodes.first
+        arr = []
+        4.times {
+          arr << Thread.new {
+            node.send_data('test', chunk) rescue nil
+          }
+        }
+        arr.each { |a| a.join }
+      end
+    end
+
+    logs = d.logs
+    assert_false(logs.any? { |log| log.include?("invalid format for PONG message") || log.include?("shared key mismatch") }, "'#{logs.last.strip}' happens")
+  end
+
   def create_target_input_driver(response_stub: nil, disconnect: false, conf: TARGET_CONFIG)
     require 'fluent/plugin/in_forward'
 
@@ -727,4 +811,113 @@ EOL
       i.configure(conf)
     end
   end
+
+  sub_test_case 'verify_connection_at_startup' do
+    test 'nodes are not available' do
+      @d = d = create_driver(CONFIG + %[
+        verify_connection_at_startup true
+        <buffer tag>
+          flush_mode immediate
+          retry_type periodic
+          retry_wait 30s
+          flush_at_shutdown false # suppress errors in d.instance_shutdown
+        </buffer>
+      ])
+      assert_raise Fluent::UnrecoverableError do
+        d.instance_start
+      end
+      d.instance_shutdown
+    end
+
+    test 'nodes_shared_key_miss_match' do
+      input_conf = TARGET_CONFIG + %[
+                   <security>
+                     self_hostname in.localhost
+                     shared_key fluentd-sharedkey
+                   </security>
+                 ]
+      target_input_driver = create_target_input_driver(conf: input_conf)
+      output_conf = %[
+        send_timeout 30
+        heartbeat_type transport
+        transport tls
+        tls_verify_hostname false
+        verify_connection_at_startup true
+        require_ack_response true
+        ack_response_timeout 5s
+        <security>
+          self_hostname localhost
+          shared_key key_miss_match
+        </security>
+        <buffer tag>
+          flush_mode immediate
+          retry_type periodic
+          retry_wait 30s
+          flush_at_shutdown false # suppress errors in d.instance_shutdown
+          flush_thread_interval 31s
+        </buffer>
+
+        <server>
+          host #{TARGET_HOST}
+          port #{TARGET_PORT}
+        </server>
+      ]
+      @d = d = create_driver(output_conf)
+
+      target_input_driver.run(expect_records: 1, timeout: 15) do
+        assert_raise Fluent::UnrecoverableError do
+          d.instance_start
+        end
+        d.instance_shutdown
+      end
+    end
+
+    test 'nodes_shared_key_match' do
+      input_conf = TARGET_CONFIG + %[
+                       <security>
+                         self_hostname in.localhost
+                         shared_key fluentd-sharedkey
+                         <client>
+                           host 127.0.0.1
+                         </client>
+                       </security>
+                     ]
+      target_input_driver = create_target_input_driver(conf: input_conf)
+
+      output_conf = %[
+          send_timeout 51
+          verify_connection_at_startup true
+          <security>
+            self_hostname localhost
+            shared_key fluentd-sharedkey
+          </security>
+          <server>
+            name test
+            host #{TARGET_HOST}
+            port #{TARGET_PORT}
+            shared_key fluentd-sharedkey
+          </server>
+      ]
+      @d = d = create_driver(output_conf)
+
+      time = event_time("2011-01-02 13:14:15 UTC")
+      records = [
+          {"a" => 1},
+          {"a" => 2}
+      ]
+
+      target_input_driver.run(expect_records: 2, timeout: 15) do
+        d.run(default_tag: 'test') do
+          records.each do |record|
+            d.feed(time, record)
+          end
+        end
+      end
+
+      events = target_input_driver.events
+      assert{ events != [] }
+      assert_equal(['test', time, records[0]], events[0])
+      assert_equal(['test', time, records[1]], events[1])
+    end
+  end
 end

data/test/plugin/test_out_secondary_file.rb

@@ -260,6 +260,20 @@ class FileOutputSecondaryTest < Test::Unit::TestCase
       assert_equal "#{TMP_DIR}/dump.bin", path
     end
 
+    test 'path with ${chunk_id}' do
+      d = create_driver %[
+        directory #{TMP_DIR}
+        basename out_file_chunk_id_${chunk_id}
+      ]
+      path = d.instance.write(@c)
+      if File.basename(path) =~ /out_file_chunk_id_([-_.@a-zA-Z0-9].*).0/
+        unique_id = Fluent::UniqueId.hex(Fluent::UniqueId.generate)
+        assert_equal unique_id.size, $1.size, "chunk_id size is mismatched"
+      else
+        flunk "chunk_id is not included in the path"
+      end
+    end
+
     data(
       invalid_tag: [/tag/, '${tag}'],
       invalid_tag0: [/tag\[0\]/, '${tag[0]}'],

data/test/plugin/test_output.rb

@@ -125,6 +125,10 @@ class OutputTest < Test::Unit::TestCase
   def create_metadata(timekey: nil, tag: nil, variables: nil)
     Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
   end
+  def create_chunk(timekey: nil, tag: nil, variables: nil)
+    m = Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
+    Fluent::Plugin::Buffer::MemoryChunk.new(m)
+  end
   def waiting(seconds)
     begin
       Timeout.timeout(seconds) do
@@ -219,7 +223,9 @@ class OutputTest < Test::Unit::TestCase
       assert @i.terminated?
     end
 
-    test '#extract_placeholders does nothing if chunk key is not specified' do
+    data(:new_api => :chunk,
+         :old_api => :metadata)
+    test '#extract_placeholders does nothing if chunk key is not specified' do |api|
       @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
       assert !@i.chunk_key_time
       assert !@i.chunk_key_tag
@@ -227,11 +233,17 @@ class OutputTest < Test::Unit::TestCase
       tmpl = "/mypath/%Y/%m/%d/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
       t = event_time('2016-04-11 20:30:00 +0900')
       v = {key1: "value1", key2: "value2"}
-      m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
-      assert_equal tmpl, @i.extract_placeholders(tmpl, m)
+      c = if api == :chunk
+            create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)
+          else
+            create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
+          end
+      assert_equal tmpl, @i.extract_placeholders(tmpl, c)
     end
 
-    test '#extract_placeholders can extract time if time key and range are configured' do
+    data(:new_api => :chunk,
+         :old_api => :metadata)
+    test '#extract_placeholders can extract time if time key and range are configured' do |api|
       @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {'timekey' => 60*30, 'timekey_zone' => "+0900"})]))
       assert @i.chunk_key_time
       assert !@i.chunk_key_tag
@@ -239,11 +251,17 @@ class OutputTest < Test::Unit::TestCase
       tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
       t = event_time('2016-04-11 20:30:00 +0900')
       v = {key1: "value1", key2: "value2"}
-      m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
-      assert_equal "/mypath/2016/04/11/20-30/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail", @i.extract_placeholders(tmpl, m)
+      c = if api == :chunk
+            create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)
+          else
+            create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
+          end
+      assert_equal "/mypath/2016/04/11/20-30/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail", @i.extract_placeholders(tmpl, c)
     end
 
-    test '#extract_placeholders can extract tag and parts of tag if tag is configured' do
+    data(:new_api => :chunk,
+         :old_api => :metadata)
+    test '#extract_placeholders can extract tag and parts of tag if tag is configured' do |api|
       @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'tag', {})]))
       assert !@i.chunk_key_time
       assert @i.chunk_key_tag
@@ -251,11 +269,17 @@ class OutputTest < Test::Unit::TestCase
       tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
       t = event_time('2016-04-11 20:30:00 +0900')
       v = {key1: "value1", key2: "value2"}
-      m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
-      assert_equal "/mypath/%Y/%m/%d/%H-%M/fluentd.test.output/test/output/${key1}/${key2}/tail", @i.extract_placeholders(tmpl, m)
+      c = if api == :chunk
+            create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)
+          else
+            create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
+          end
+      assert_equal "/mypath/%Y/%m/%d/%H-%M/fluentd.test.output/test/output/${key1}/${key2}/tail", @i.extract_placeholders(tmpl, c)
     end
 
-    test '#extract_placeholders can extract variables if variables are configured' do
+    data(:new_api => :chunk,
+         :old_api => :metadata)
+    test '#extract_placeholders can extract variables if variables are configured' do |api|
       @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'key1,key2', {})]))
       assert !@i.chunk_key_time
       assert !@i.chunk_key_tag
@@ -263,32 +287,95 @@ class OutputTest < Test::Unit::TestCase
       tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
       t = event_time('2016-04-11 20:30:00 +0900')
       v = {key1: "value1", key2: "value2"}
-      m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
-      assert_equal "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/value1/value2/tail", @i.extract_placeholders(tmpl, m)
+      c = if api == :chunk
+            create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)
+          else
+            create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
+          end
+      assert_equal "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/value1/value2/tail", @i.extract_placeholders(tmpl, c)
+    end
+
+    data(:new_api => :chunk,
+         :old_api => :metadata)
+    test '#extract_placeholders can extract nested variables if variables are configured with dot notation' do |api|
+      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'key,$.nest.key', {})]))
+      assert !@i.chunk_key_time
+      assert !@i.chunk_key_tag
+      assert_equal ['key','$.nest.key'], @i.chunk_keys
+      tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key}/${$.nest.key}/tail"
+      t = event_time('2016-04-11 20:30:00 +0900')
+      v = {:key => "value1", :"$.nest.key" => "value2"}
+      c = if api == :chunk
+            create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)
+          else
+            create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
+          end
+      assert_equal "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/value1/value2/tail", @i.extract_placeholders(tmpl, c)
+    end
+
+    data(:new_api => :chunk,
+         :old_api => :metadata)
+    test '#extract_placeholders can extract all chunk keys if configured' do |api|
+      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,key1,key2', {'timekey' => 60*30, 'timekey_zone' => "+0900"})]))
+      assert @i.chunk_key_time
+      assert @i.chunk_key_tag
+      assert_equal ['key1','key2'], @i.chunk_keys
+      tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
+      t = event_time('2016-04-11 20:30:00 +0900')
+      v = {key1: "value1", key2: "value2"}
+      c = if api == :chunk
+            create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)
+          else
+            create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
+          end
+      assert_equal "/mypath/2016/04/11/20-30/fluentd.test.output/test/output/value1/value2/tail", @i.extract_placeholders(tmpl, c)
     end
 
-    test '#extract_placeholders can extract all chunk keys if configured' do
+    data(:new_api => :chunk,
+         :old_api => :metadata)
+    test '#extract_placeholders can extract negative index with tag' do |api|
       @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,key1,key2', {'timekey' => 60*30, 'timekey_zone' => "+0900"})]))
       assert @i.chunk_key_time
       assert @i.chunk_key_tag
       assert_equal ['key1','key2'], @i.chunk_keys
-      tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
+      tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[-1]}/${tag[-2]}/${key1}/${key2}/tail"
       t = event_time('2016-04-11 20:30:00 +0900')
       v = {key1: "value1", key2: "value2"}
-      m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
-      assert_equal "/mypath/2016/04/11/20-30/fluentd.test.output/test/output/value1/value2/tail", @i.extract_placeholders(tmpl, m)
+      c = if api == :chunk
+            create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)
+          else
+            create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
+          end
+      assert_equal "/mypath/2016/04/11/20-30/fluentd.test.output/output/test/value1/value2/tail", @i.extract_placeholders(tmpl, c)
     end
 
-    test '#extract_placeholders removes out-of-range tag part and unknown variable placeholders' do
+    data(:new_api => :chunk,
+         :old_api => :metadata)
+    test '#extract_placeholders removes out-of-range tag part and unknown variable placeholders' do |api|
       @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,key1,key2', {'timekey' => 60*30, 'timekey_zone' => "+0900"})]))
       assert @i.chunk_key_time
       assert @i.chunk_key_tag
       assert_equal ['key1','key2'], @i.chunk_keys
-      tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[3]}/${tag[4]}/${key3}/${key4}/tail"
+      tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[3]}/${tag[-4]}/${key3}/${key4}/tail"
+      t = event_time('2016-04-11 20:30:00 +0900')
+      v = {key1: "value1", key2: "value2"}
+      c = if api == :chunk
+            create_chunk(timekey: t, tag: 'fluentd.test.output', variables: v)
+          else
+            create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
+          end
+      assert_equal "/mypath/2016/04/11/20-30/fluentd.test.output/////tail", @i.extract_placeholders(tmpl, c)
+    end
+
+    test '#extract_placeholders logs warn message if metadata is passed for ${chunk_id} placeholder' do
+      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
+      tmpl = "/mypath/${chunk_id}/tail"
       t = event_time('2016-04-11 20:30:00 +0900')
       v = {key1: "value1", key2: "value2"}
       m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
-      assert_equal "/mypath/2016/04/11/20-30/fluentd.test.output/////tail", @i.extract_placeholders(tmpl, m)
+      @i.extract_placeholders(tmpl, m)
+      logs = @i.log.out.logs
+      assert { logs.any? { |log| log.include?("${chunk_id} is not allowed in this plugin") } }
     end
 
     sub_test_case '#placeholder_validators' do
@@ -372,7 +459,7 @@ class OutputTest < Test::Unit::TestCase
     end
 
     sub_test_case '#placeholder_validate!' do
-      test 'raises configuration error for a templace when timestamp placeholders exist but time key is missing' do
+      test 'raises configuration error for a template when timestamp placeholders exist but time key is missing' do
         @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
         assert_raise Fluent::ConfigError.new("Parameter 'path: /path/without/timestamp/file.%Y%m%d-%H%M.log' has timestamp placeholders, but chunk key 'time' is not configured") do
           @i.placeholder_validate!(:path, "/path/without/timestamp/file.%Y%m%d-%H%M.log")
@@ -414,6 +501,9 @@ class OutputTest < Test::Unit::TestCase
         assert_nothing_raised do
           @i.placeholder_validate!(:path, "/my/path/${tag}/file.${tag[2]}.log")
         end
+        assert_nothing_raised do
+          @i.placeholder_validate!(:path, "/my/path/${tag}/file.${tag[-1]}.log")
+        end
       end
 
       test 'raises configuration error for a template when variable key placeholders exist but chunk keys are missing' do
@@ -493,11 +583,21 @@ class OutputTest < Test::Unit::TestCase
       assert_equal ['.hidden', '0001', '@timestamp', 'a_key', 'my-domain'], @i.get_placeholders_keys("http://${my-domain}/${.hidden}/${0001}/${a_key}?timestamp=${@timestamp}")
     end
 
+    data('include space' => 'ke y',
+         'bracket notation' => "$['key']",
+         'invalid notation' => "$.ke y")
+    test 'configure checks invalid chunk keys' do |chunk_keys|
+      i = create_output(:buffered)
+      assert_raise Fluent::ConfigError do
+        i.configure(config_element('ROOT' , '', {}, [config_element('buffer', chunk_keys)]))
+      end
+    end
+
     test '#metadata returns object which contains tag/timekey/variables from records as specified in configuration' do
       tag = 'test.output'
       time = event_time('2016-04-12 15:31:23 -0700')
       timekey = event_time('2016-04-12 15:00:00 -0700')
-      record = {"key1" => "value1", "num1" => 1, "message" => "my message"}
+      record = {"key1" => "value1", "num1" => 1, "message" => "my message", "nest" => {"key" => "nested value"}}
 
       i1 = create_output(:buffered)
       i1.configure(config_element('ROOT','',{},[config_element('buffer', '')]))
@@ -530,6 +630,10 @@ class OutputTest < Test::Unit::TestCase
       i8 = create_output(:buffered)
       i8.configure(config_element('ROOT','',{},[config_element('buffer', 'time,tag,key1', {"timekey" => 3600, "timekey_zone" => "-0700"})]))
       assert_equal create_metadata(timekey: timekey, tag: tag, variables: {key1: "value1"}), i8.metadata(tag, time, record)
+
+      i9 = create_output(:buffered)
+      i9.configure(config_element('ROOT','',{},[config_element('buffer', 'key1,$.nest.key', {})]))
+      assert_equal create_metadata(variables: {:key1 => "value1", :"$.nest.key" => 'nested value'}), i9.metadata(tag, time, record)
     end
 
     test '#emit calls #process via #emit_sync for non-buffered output' do
@@ -724,34 +828,43 @@ class OutputTest < Test::Unit::TestCase
       assert_equal :interval, @i.instance_variable_get(:@flush_mode)
     end
 
-    test "Warn if primary type is different from secondary type and either primary or secondary has custom_format" do
-      o = create_output(:buffered)
-      mock(o.log).warn("secondary type should be same with primary one",
-                                { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" })
+    sub_test_case 'configure secondary' do
+      test "Warn if primary type is different from secondary type and either primary or secondary has custom_format" do
+        o = create_output(:buffered)
+        mock(o.log).warn("secondary type should be same with primary one",
+                         { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" })
 
-      o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'test', 'name' => "cool"})]))
-      assert_not_nil o.instance_variable_get(:@secondary)
-    end
+        o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'test', 'name' => "cool"})]))
+        assert_not_nil o.instance_variable_get(:@secondary)
+      end
 
-    test "don't warn if primary type is the same as secondary type" do
-      o = Fluent::Plugin::TestOutput.new
-      mock(o.log).warn("secondary type should be same with primary one",
-                                { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" }).never
+      test "don't warn if primary type is the same as secondary type" do
+        o = Fluent::Plugin::TestOutput.new
+        mock(o.log).warn("secondary type should be same with primary one",
+                         { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" }).never
 
-      o.configure(config_element('ROOT','',{'name' => "cool2"},
-                                 [config_element('secondary','',{'@type'=>'test', 'name' => "cool"}),
-                                  config_element('buffer','',{'@type'=>'memory'})]
-                                ))
-      assert_not_nil o.instance_variable_get(:@secondary)
-    end
+        o.configure(config_element('ROOT','',{'name' => "cool2"},
+                                   [config_element('secondary','',{'@type'=>'test', 'name' => "cool"}),
+                                    config_element('buffer','',{'@type'=>'memory'})]
+                                  ))
+        assert_not_nil o.instance_variable_get(:@secondary)
+      end
 
-    test "don't warn if primary type is different from secondary type and both don't have custom_format" do
-      o = create_output(:standard)
-      mock(o.log).warn("secondary type should be same with primary one",
-                                { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" }).never
+      test "don't warn if primary type is different from secondary type and both don't have custom_format" do
+        o = create_output(:standard)
+        mock(o.log).warn("secondary type should be same with primary one",
+                         { primary: o.class.to_s, secondary: "Fluent::Plugin::TestOutput" }).never
 
-      o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'test', 'name' => "cool"})]))
-      assert_not_nil o.instance_variable_get(:@secondary)
+        o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'test', 'name' => "cool"})]))
+        assert_not_nil o.instance_variable_get(:@secondary)
+      end
+
+      test "raise configuration error if secondary type specifies non buffered output" do
+        o = create_output(:standard)
+        assert_raise Fluent::ConfigError do
+          o.configure(config_element('ROOT','',{},[config_element('secondary','',{'@type'=>'copy'})]))
+        end
+      end
     end
   end
 
@@ -824,7 +937,8 @@ class OutputTest < Test::Unit::TestCase
 
   sub_test_case 'slow_flush_log_threshold' do
     def invoke_slow_flush_log_threshold_test(i)
-      i.configure(config_element('ROOT', '', {'slow_flush_log_threshold' => 0.5}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
+      i.configure(config_element('ROOT', '', {'slow_flush_log_threshold' => 0.5},
+                                 [config_element('buffer', '', {"flush_mode" => "immediate", "flush_thread_interval" => 30})]))
       i.start
       i.after_start