fluentd 0.14.8 → 0.14.9

data/test/plugin/test_out_exec.rb

@@ -18,123 +18,211 @@ class ExecOutputTest < Test::Unit::TestCase
 
   TMP_DIR = File.dirname(__FILE__) + "/../tmp/out_exec#{ENV['TEST_ENV_NUMBER']}"
 
-  CONFIG = %[
+  def create_driver(config)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::ExecOutput).configure(config)
+  end
+
+  def create_test_data
+    time = event_time("2011-01-02 13:14:15.123")
+    records = [{"k1"=>"v1","kx"=>"vx"}, {"k1"=>"v2","kx"=>"vx"}]
+    return time, records
+  end
+
+  DEFAULT_CONFIG_ONLY_WITH_KEYS = %[
+    command cat >#{TMP_DIR}/out
+    <format>
+      keys ["k1", "kx"]
+    </format>
+  ]
+
+  test 'configure in default' do
+    d = create_driver DEFAULT_CONFIG_ONLY_WITH_KEYS
+    assert{ d.instance.formatter.is_a? Fluent::Plugin::TSVFormatter }
+    assert_equal ["k1", "kx"], d.instance.formatter.keys
+    assert_nil d.instance.inject_config
+  end
+
+  TSV_CONFIG = %[
+    command cat >#{TMP_DIR}/out
+    <inject>
+      tag_key tag
+      time_key time
+      time_format %Y-%m-%d %H:%M:%S
+      localtime yes
+    </inject>
+    <format>
+      @type tsv
+      keys time, tag, k1
+    </format>
+  ]
+  TSV_CONFIG_WITH_SUBSEC = %[
+    command cat >#{TMP_DIR}/out
+    <inject>
+      tag_key tag
+      time_key time
+      time_format %Y-%m-%d %H:%M:%S.%3N
+      localtime yes
+    </inject>
+    <format>
+      @type tsv
+      keys time, tag, k1
+    </format>
+  ]
+  TSV_CONFIG_WITH_BUFFER = TSV_CONFIG + %[
+    <buffer time>
+      @type memory
+      timekey 3600
+      flush_thread_count 5
+      chunk_limit_size 50m
+      total_limit_size #{50 * 1024 * 1024 * 128}
+      flush_at_shutdown yes
+    </buffer>
+  ]
+  JSON_CONFIG = %[
+    command cat >#{TMP_DIR}/out
+    <format>
+      @type json
+    </format>
+  ]
+  MSGPACK_CONFIG = %[
+    command cat >#{TMP_DIR}/out
+    <format>
+      @type msgpack
+    </format>
+  ]
+
+  CONFIG_COMPAT = %[
     buffer_path #{TMP_DIR}/buffer
     command cat >#{TMP_DIR}/out
     localtime
   ]
-  TSV_CONFIG = %[
+  TSV_CONFIG_COMPAT = %[
     keys "time,tag,k1"
     tag_key "tag"
     time_key "time"
     time_format %Y-%m-%d %H:%M:%S
   ]
+  BUFFER_CONFIG_COMPAT = %[
+    buffer_type memory
+    time_slice_format %Y%m%d%H
+    num_threads 5
+    buffer_chunk_limit 50m
+    buffer_queue_limit 128
+    flush_at_shutdown yes
+  ]
+  TSV_CONFIG_WITH_SUBSEC_COMPAT = %[
+    keys "time,tag,k1"
+    tag_key "tag"
+    time_key "time"
+    time_format %Y-%m-%d %H:%M:%S.%3N
+  ]
 
-  def create_driver(conf = TSV_CONFIG)
-    config = CONFIG + conf
-    Fluent::Test::Driver::Output.new(Fluent::Plugin::ExecOutput).configure(config)
-  end
-
-  def create_test_case
-    time = Time.parse("2011-01-02 13:14:15").to_i
-    tests = [{"k1"=>"v1","kx"=>"vx"}, {"k1"=>"v2","kx"=>"vx"}]
-    return time, tests
-  end
-
-  def test_configure
-    d = create_driver
+  data(
+    'with sections' => TSV_CONFIG,
+    'traditional' => CONFIG_COMPAT + TSV_CONFIG_COMPAT,
+  )
+  test 'configure for tsv' do |conf|
+    d = create_driver(conf)
 
-    assert_equal ["time","tag","k1"], d.instance.keys
-    assert_equal "tag", d.instance.tag_key
-    assert_equal "time", d.instance.time_key
-    assert_equal "%Y-%m-%d %H:%M:%S", d.instance.time_format
-    assert_equal true, d.instance.localtime
+    assert_equal ["time","tag","k1"], d.instance.formatter.keys
+    assert_equal "tag", d.instance.inject_config.tag_key
+    assert_equal "time", d.instance.inject_config.time_key
+    assert_equal "%Y-%m-%d %H:%M:%S", d.instance.inject_config.time_format
+    assert_equal true, d.instance.inject_config.localtime
   end
 
-  def test_configure_with_compat_buffer_parameters
-    conf = TSV_CONFIG + %[
-      buffer_type memory
-      time_slice_format %Y%m%d%H
-      num_threads 5
-      buffer_chunk_limit 50m
-      buffer_queue_limit 128
-      flush_at_shutdown yes
-    ]
+  data(
+    'with sections' => TSV_CONFIG_WITH_BUFFER,
+    'traditional' => CONFIG_COMPAT + TSV_CONFIG_COMPAT + BUFFER_CONFIG_COMPAT,
+  )
+  test 'configure_with_compat_buffer_parameters' do |conf|
     d = create_driver(conf)
     assert_equal 3600, d.instance.buffer_config.timekey
     assert_equal 5, d.instance.buffer_config.flush_thread_count
     assert_equal 50*1024*1024, d.instance.buffer.chunk_limit_size
-    assert_equal 128, d.instance.buffer.queue_length_limit
+    assert_equal 50*1024*1024*128, d.instance.buffer.total_limit_size
     assert d.instance.buffer_config.flush_at_shutdown
   end
 
-  def test_format
-    d = create_driver
-    time, tests = create_test_case
+  data(
+    'with sections' => TSV_CONFIG,
+    'traditional' => CONFIG_COMPAT + TSV_CONFIG_COMPAT,
+  )
+  test 'format' do |conf|
+    d = create_driver(conf)
+    time, records = create_test_data
 
     d.run(default_tag: 'test') do
-      d.feed(time, tests[0])
-      d.feed(time, tests[1])
+      d.feed(time, records[0])
+      d.feed(time, records[1])
     end
 
     assert_equal %[2011-01-02 13:14:15\ttest\tv1\n], d.formatted[0]
     assert_equal %[2011-01-02 13:14:15\ttest\tv2\n], d.formatted[1]
   end
 
-  def test_format_json
-    d = create_driver("format json")
-    time, tests = create_test_case
+  data(
+    'with sections' => JSON_CONFIG,
+    'traditional' => CONFIG_COMPAT + "format json",
+  )
+  test 'format_json' do |conf|
+    d = create_driver(conf)
+    time, records = create_test_data
 
     d.run(default_tag: 'test') do
-      d.feed(time, tests[0])
-      d.feed(time, tests[1])
+      d.feed(time, records[0])
+      d.feed(time, records[1])
     end
 
-    assert_equal Yajl.dump(tests[0]) + "\n", d.formatted[0]
-    assert_equal Yajl.dump(tests[1]) + "\n", d.formatted[1]
+    assert_equal Yajl.dump(records[0]) + "\n", d.formatted[0]
+    assert_equal Yajl.dump(records[1]) + "\n", d.formatted[1]
   end
 
-  def test_format_msgpack
-    d = create_driver("format msgpack")
-    time, tests = create_test_case
+  data(
+    'with sections' => MSGPACK_CONFIG,
+    'traditional' => CONFIG_COMPAT + "format msgpack"
+  )
+  test 'format_msgpack' do |conf|
+    d = create_driver(conf)
+    time, records = create_test_data
 
     d.run(default_tag: 'test') do
-      d.feed(time, tests[0])
-      d.feed(time, tests[1])
+      d.feed(time, records[0])
+      d.feed(time, records[1])
     end
 
-    assert_equal tests[0].to_msgpack, d.formatted[0]
-    assert_equal tests[1].to_msgpack, d.formatted[1]
+    assert_equal records[0].to_msgpack, d.formatted[0]
+    assert_equal records[1].to_msgpack, d.formatted[1]
   end
 
-  def test_format_time
-    config = %[
-      keys "time,tag,k1"
-      tag_key "tag"
-      time_key "time"
-      time_format %Y-%m-%d %H:%M:%S.%3N
-    ]
-    d = create_driver(config)
-
-    time = event_time("2011-01-02 13:14:15.123")
-    tests = [{"k1"=>"v1","kx"=>"vx"}, {"k1"=>"v2","kx"=>"vx"}]
+  data(
+    'with sections' => TSV_CONFIG_WITH_SUBSEC,
+    'traditional' => CONFIG_COMPAT + TSV_CONFIG_WITH_SUBSEC_COMPAT,
+  )
+  test 'format subsecond time' do |conf|
+    d = create_driver(conf)
+    time, records = create_test_data
 
     d.run(default_tag: 'test') do
-      d.feed(time, tests[0])
-      d.feed(time, tests[1])
+      d.feed(time, records[0])
+      d.feed(time, records[1])
     end
 
     assert_equal %[2011-01-02 13:14:15.123\ttest\tv1\n], d.formatted[0]
     assert_equal %[2011-01-02 13:14:15.123\ttest\tv2\n], d.formatted[1]
   end
 
-  def test_write
-    d = create_driver
-    time, tests = create_test_case
+  data(
+    'with sections' => TSV_CONFIG,
+    'traditional' => CONFIG_COMPAT + TSV_CONFIG_COMPAT,
+  )
+  test 'write' do |conf|
+    d = create_driver(conf)
+    time, records = create_test_data
 
     d.run(default_tag: 'test', flush: true) do
-      d.feed(time, tests[0])
-      d.feed(time, tests[1])
+      d.feed(time, records[0])
+      d.feed(time, records[1])
     end
 
     expect_path = "#{TMP_DIR}/out"
@@ -151,5 +239,72 @@ class ExecOutputTest < Test::Unit::TestCase
       %[2011-01-02 13:14:15\ttest\tv2\n]
     assert_equal expect_data, data
   end
-end
 
+  sub_test_case 'when executed process dies unexpectedly' do
+    setup do
+      @gen_config = ->(num){ <<EOC
+    command ruby -e "ARGV.first.to_i == 0 ? open(ARGV[1]){|f| STDOUT.write f.read} : (sleep 1 ; exit ARGV.first.to_i)" #{num} >#{TMP_DIR}/fail_out
+    <inject>
+      tag_key tag
+      time_key time
+      time_format %Y-%m-%d %H:%M:%S
+      localtime yes
+    </inject>
+    <format>
+      @type tsv
+      keys time, tag, k1
+    </format>
+EOC
+      }
+    end
+
+    test 'flushed chunk will be committed after child process successfully exits' do
+      d = create_driver(@gen_config.call(0))
+      time, records = create_test_data
+
+      expect_path = "#{TMP_DIR}/fail_out"
+
+      d.end_if{ File.exist?(expect_path) }
+      d.run(default_tag: 'test', flush: true, wait_flush_completion: false, shutdown: false) do
+        d.feed(time, records[0])
+        d.feed(time, records[1])
+      end
+
+      assert{ File.exist?(expect_path) }
+
+      data = File.read(expect_path)
+      expect_data =
+        %[2011-01-02 13:14:15\ttest\tv1\n] +
+        %[2011-01-02 13:14:15\ttest\tv2\n]
+      assert_equal expect_data, data
+
+      assert{ d.instance.buffer.queue.empty? }
+      assert{ d.instance.dequeued_chunks.empty? }
+
+      d.instance_shutdown
+    end
+
+    test 'flushed chunk will be taken back after child process unexpectedly exits' do
+      d = create_driver(@gen_config.call(3))
+      time, records = create_test_data
+
+      expect_path = "#{TMP_DIR}/fail_out"
+
+      d.end_if{ d.instance.log.out.logs.any?{|line| line.include?("command exits with error code") } }
+      d.run(default_tag: 'test', flush: true, wait_flush_completion: false, shutdown: false) do
+        d.feed(time, records[0])
+        d.feed(time, records[1])
+      end
+
+      assert{ d.instance.dequeued_chunks.empty? } # because it's already taken back
+      assert{ d.instance.buffer.queue.size == 1 }
+
+      logs = d.instance.log.out.logs
+      assert{ logs.any?{|line| line.include?("command exits with error code") && line.include?("status=3") } }
+
+      assert{ File.exist?(expect_path) && File.size(expect_path) == 0 }
+
+      d.instance_shutdown
+    end
+  end
+end

data/test/plugin/test_out_exec_filter.rb

@@ -1,5 +1,5 @@
 require_relative '../helper'
-require 'fluent/test'
+require 'fluent/test/driver/output'
 require 'fluent/plugin/out_exec_filter'
 require 'fileutils'
 
@@ -9,6 +9,29 @@ class ExecFilterOutputTest < Test::Unit::TestCase
   end
 
   CONFIG = %[
+    command cat
+    num_children 3
+    <inject>
+      tag_key     tag
+      time_key    time_in
+      time_type   string
+      time_format %Y-%m-%d %H:%M:%S
+    </inject>
+    <format>
+      keys ["time_in", "tag", "k1"]
+    </format>
+    <parse>
+      keys ["time_out", "tag", "k2"]
+    </parse>
+    <extract>
+      tag_key     tag
+      time_key    time_out
+      time_type   string
+      time_format %Y-%m-%d %H:%M:%S
+    </extract>
+  ]
+
+  CONFIG_COMPAT = %[
     command cat
     in_keys time_in,tag,k1
     out_keys time_out,tag,k2
@@ -20,35 +43,34 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     num_children 3
   ]
 
-  def create_driver(conf = CONFIG, tag = 'test')
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::ExecFilterOutput, tag).configure(conf)
-  end
-
-  def sed_unbuffered_support?
-    @sed_unbuffered_support ||= lambda {
-      system("echo xxx | sed --unbuffered -l -e 's/x/y/g' >#{IO::NULL} 2>&1")
-      $?.success?
-    }.call
-  end
-
-  def sed_unbuffered_option
-    sed_unbuffered_support? ? '--unbuffered' : ''
+  def create_driver(conf)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::ExecFilterOutput).configure(conf)
   end
 
-  def test_configure
-    d = create_driver
-
-    assert d.instance.instance_eval{ @overrides_format_stream }
-
-    assert_equal ["time_in","tag","k1"], d.instance.in_keys
-    assert_equal ["time_out","tag","k2"], d.instance.out_keys
-    assert_equal "tag", d.instance.out_tag_key
-    assert_equal "tag", d.instance.in_tag_key
-    assert_equal "time_in", d.instance.in_time_key
-    assert_equal "time_out", d.instance.out_time_key
-    assert_equal "%Y-%m-%d %H:%M:%S", d.instance.in_time_format
-    assert_equal "%Y-%m-%d %H:%M:%S", d.instance.out_time_format
-    assert_equal true, d.instance.localtime
+  SED_SUPPORT_UNBUFFERED_OPTION = ->(){
+    system("echo xxx | sed --unbuffered -l -e 's/x/y/g' >#{IO::NULL} 2>&1")
+    $?.success?
+  }.call
+  SED_UNBUFFERED_OPTION = SED_SUPPORT_UNBUFFERED_OPTION ? '--unbuffered' : ''
+
+  data(
+    'with sections' => CONFIG,
+    'traditional' => CONFIG_COMPAT,
+  )
+  test 'configure' do |conf|
+    d = create_driver(conf)
+
+    assert_false d.instance.parser.estimate_current_event
+
+    assert_equal ["time_in","tag","k1"], d.instance.formatter.keys
+    assert_equal ["time_out","tag","k2"], d.instance.parser.keys
+    assert_equal "tag", d.instance.inject_config.tag_key
+    assert_equal "tag", d.instance.extract_config.tag_key
+    assert_equal "time_in", d.instance.inject_config.time_key
+    assert_equal "time_out", d.instance.extract_config.time_key
+    assert_equal "%Y-%m-%d %H:%M:%S", d.instance.inject_config.time_format
+    assert_equal "%Y-%m-%d %H:%M:%S", d.instance.extract_config.time_format
+    assert_equal true, d.instance.inject_config.localtime
     assert_equal 3, d.instance.num_children
 
     d = create_driver %[
@@ -61,7 +83,7 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     ]
     assert_equal "sed -l -e s/foo/bar/", d.instance.command
 
-    d = create_driver(CONFIG + %[
+    d = create_driver(conf + %[
       remove_prefix before
       add_prefix after
     ])
@@ -69,194 +91,523 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     assert_equal "after" , d.instance.add_prefix
   end
 
-  def test_emit_1
-    d = create_driver
-
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15")
-
-    d.expected_emits_length = 2
-    d.run do
-      d.emit({"k1"=>1}, time)
-      d.emit({"k1"=>2}, time)
+  data(
+    'with sections' => CONFIG,
+    'traditional' => CONFIG_COMPAT,
+  )
+  test 'emit events with TSV format' do |conf|
+    d = create_driver(conf)
+    time = event_time("2011-01-02 13:14:15")
+
+    d.run(default_tag: 'test', expect_emits: 2, timeout: 10) do
+      # sleep 0.1 until d.instance.children && !d.instance.children.empty? && d.instance.children.all?{|c| c.finished == false }
+      d.feed(time, {"k1"=>1})
+      d.feed(time, {"k1"=>2})
     end
 
-    emits = d.emits
-    assert_equal 2, emits.length
-    assert_equal ["test", time, {"k2"=>"1"}], emits[0]
-    assert_equal_event_time time, emits[0][1]
-    assert_equal ["test", time, {"k2"=>"2"}], emits[1]
-    assert_equal_event_time time, emits[1][1]
+    assert_equal "2011-01-02 13:14:15\ttest\t1\n", d.formatted[0]
+    assert_equal "2011-01-02 13:14:15\ttest\t2\n", d.formatted[1]
+
+    events = d.events
+    assert_equal 2, events.length
+    assert_equal_event_time time, events[0][1]
+    assert_equal ["test", time, {"k2"=>"1"}], events[0]
+    assert_equal_event_time time, events[1][1]
+    assert_equal ["test", time, {"k2"=>"2"}], events[1]
   end
 
-  def test_emit_2
-    d = create_driver %[
-      command cat
-      in_keys time,k1
-      out_keys time,k2
-      tag xxx
+  CONFIG_WITHOUT_TIME_FORMAT = %[
+    command cat
+    num_children 3
+    tag xxx
+    <inject>
       time_key time
-      num_children 3
-    ]
-
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15")
+      time_type unixtime
+    </inject>
+    <format>
+      keys time,k1
+    </format>
+    <parse>
+      keys time,k2
+      time_key time
+      time_type unixtime
+    </parse>
+  ]
+  CONFIG_WITHOUT_TIME_FORMAT_COMPAT = %[
+    command cat
+    in_keys time,k1
+    out_keys time,k2
+    tag xxx
+    time_key time
+    num_children 3
+  ]
 
-    d.expected_emits_length = 2
-    d.run do
-      d.emit({"k1"=>1}, time)
-      d.emit({"k1"=>2}, time)
+  data(
+    'with sections' => CONFIG_WITHOUT_TIME_FORMAT,
+    'traditional' => CONFIG_WITHOUT_TIME_FORMAT_COMPAT,
+  )
+  test 'emit events without time format configuration' do |conf|
+    d = create_driver(conf)
+    time = event_time("2011-01-02 13:14:15 +0900")
+
+    d.run(default_tag: 'test', expect_emits: 2, timeout: 10) do
+      d.feed(time, {"k1"=>1})
+      d.feed(time, {"k1"=>2})
     end
 
-    emits = d.emits
-    assert_equal 2, emits.length
-    assert_equal ["xxx", time, {"k2"=>"1"}], emits[0]
-    assert_equal_event_time time, emits[0][1]
-    assert_equal ["xxx", time, {"k2"=>"2"}], emits[1]
-    assert_equal_event_time time, emits[1][1]
+    assert_equal "1293941655\t1\n", d.formatted[0]
+    assert_equal "1293941655\t2\n", d.formatted[1]
+
+    events = d.events
+    assert_equal 2, events.length
+    assert_equal_event_time time, events[0][1]
+    assert_equal ["xxx", time, {"k2"=>"1"}], events[0]
+    assert_equal_event_time time, events[1][1]
+    assert_equal ["xxx", time, {"k2"=>"2"}], events[1]
   end
 
-  def test_emit_3
-    d = create_driver %[
-      command grep --line-buffered -v poo
-      in_keys time,val1
-      out_keys time,val2
-      tag xxx
+  CONFIG_TO_DO_GREP = %[
+    command grep --line-buffered -v poo
+    num_children 3
+    tag xxx
+    <inject>
       time_key time
-      num_children 3
-    ]
-
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15")
+      time_type unixtime
+    </inject>
+    <format>
+      keys time, val1
+    </format>
+    <parse>
+      keys time, val2
+      time_key time
+      time_type unixtime
+    </parse>
+  ]
+  CONFIG_TO_DO_GREP_COMPAT = %[
+    command grep --line-buffered -v poo
+    in_keys time,val1
+    out_keys time,val2
+    tag xxx
+    time_key time
+    num_children 3
+  ]
 
-    d.expected_emits_length = 2
-    d.run do
-      d.emit({"val1"=>"sed-ed value foo"}, time)
-      d.emit({"val1"=>"sed-ed value poo"}, time)
+  data(
+    'with sections' => CONFIG_TO_DO_GREP,
+    'traditional' => CONFIG_TO_DO_GREP_COMPAT,
+  )
+  test 'emit events through grep command' do |conf|
+    d = create_driver(conf)
+    time = event_time("2011-01-02 13:14:15 +0900")
+
+    d.run(default_tag: 'test', expect_emits: 1, timeout: 10) do
+      d.feed(time, {"val1"=>"sed-ed value poo"})
+      d.feed(time, {"val1"=>"sed-ed value foo"})
     end
 
-    emits = d.emits
-    assert_equal 1, emits.length
-    assert_equal ["xxx", time, {"val2"=>"sed-ed value foo"}], emits[0]
-    assert_equal_event_time time, emits[0][1]
+    assert_equal "1293941655\tsed-ed value poo\n", d.formatted[0]
+    assert_equal "1293941655\tsed-ed value foo\n", d.formatted[1]
 
-    d = create_driver %[
-      command sed #{sed_unbuffered_option} -l -e s/foo/bar/
-      in_keys time,val1
-      out_keys time,val2
-      tag xxx
-      time_key time
-      num_children 3
-    ]
+    events = d.events
+    assert_equal 1, events.length
+    assert_equal_event_time time, events[0][1]
+    assert_equal ["xxx", time, {"val2"=>"sed-ed value foo"}], events[0]
+  end
 
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15")
+  CONFIG_TO_DO_SED = %[
+    command sed #{SED_UNBUFFERED_OPTION} -l -e s/foo/bar/
+    num_children 3
+    tag xxx
+    <inject>
+      time_key time
+      time_type unixtime
+    </inject>
+    <format>
+      keys time, val1
+    </format>
+    <parse>
+      keys time, val2
+      time_key time
+      time_type unixtime
+    </parse>
+  ]
+  CONFIG_TO_DO_SED_COMPAT = %[
+    command sed #{SED_UNBUFFERED_OPTION} -l -e s/foo/bar/
+    in_keys time,val1
+    out_keys time,val2
+    tag xxx
+    time_key time
+    num_children 3
+  ]
 
-    d.expected_emits_length = 2
-    d.run do
-      d.emit({"val1"=>"sed-ed value foo"}, time)
-      d.emit({"val1"=>"sed-ed value poo"}, time)
+  data(
+    'with sections' => CONFIG_TO_DO_SED,
+    'traditional' => CONFIG_TO_DO_SED_COMPAT,
+  )
+  test 'emit events through sed command' do |conf|
+    d = create_driver(conf)
+    time = event_time("2011-01-02 13:14:15 +0900")
+
+    d.run(default_tag: 'test', expect_emits: 1, timeout: 10) do
+      d.feed(time, {"val1"=>"sed-ed value poo"})
+      d.feed(time, {"val1"=>"sed-ed value foo"})
     end
 
-    emits = d.emits
-    assert_equal 2, emits.length
-    assert_equal ["xxx", time, {"val2"=>"sed-ed value bar"}], emits[0]
-    assert_equal_event_time time, emits[0][1]
-    assert_equal ["xxx", time, {"val2"=>"sed-ed value poo"}], emits[1]
-    assert_equal_event_time time, emits[1][1]
+    assert_equal "1293941655\tsed-ed value poo\n", d.formatted[0]
+    assert_equal "1293941655\tsed-ed value foo\n", d.formatted[1]
+
+    events = d.events
+    assert_equal 2, events.length
+    assert_equal_event_time time, events[0][1]
+    assert_equal ["xxx", time, {"val2"=>"sed-ed value poo"}], events[0]
+    assert_equal_event_time time, events[1][1]
+    assert_equal ["xxx", time, {"val2"=>"sed-ed value bar"}], events[1]
   end
 
-  def test_emit_4
-    d = create_driver(%[
-      command sed #{sed_unbuffered_option} -l -e s/foo/bar/
-      in_keys tag,time,val1
-      remove_prefix input
-      out_keys tag,time,val2
-      add_prefix output
+  CONFIG_TO_DO_SED_WITH_TAG_MODIFY = %[
+    command sed #{SED_UNBUFFERED_OPTION} -l -e s/foo/bar/
+    num_children 3
+    remove_prefix input
+    add_prefix output
+    <inject>
       tag_key tag
       time_key time
-      num_children 3
-    ], 'input.test')
+    </inject>
+    <format>
+      keys tag, time, val1
+    </format>
+    <parse>
+      keys tag, time, val2
+    </parse>
+    <extract>
+      tag_key tag
+      time_key time
+    </extract>
+  ]
+  CONFIG_TO_DO_SED_WITH_TAG_MODIFY_COMPAT = %[
+    command sed #{SED_UNBUFFERED_OPTION} -l -e s/foo/bar/
+    in_keys tag,time,val1
+    remove_prefix input
+    out_keys tag,time,val2
+    add_prefix output
+    tag_key tag
+    time_key time
+    num_children 3
+  ]
+
+  data(
+    'with sections' => CONFIG_TO_DO_SED_WITH_TAG_MODIFY,
+    'traditional' => CONFIG_TO_DO_SED_WITH_TAG_MODIFY_COMPAT,
+  )
+  test 'emit events with add/remove tag prefix' do |conf|
+    d = create_driver(conf)
 
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15")
+    time = event_time("2011-01-02 13:14:15 +0900")
 
-    d.expected_emits_length = 2
-    d.run do
-      d.emit({"val1"=>"sed-ed value foo"}, time)
-      d.emit({"val1"=>"sed-ed value poo"}, time)
+    d.run(default_tag: 'input.test', expect_emits: 2, timeout: 10) do
+      d.feed(time, {"val1"=>"sed-ed value foo"})
+      d.feed(time, {"val1"=>"sed-ed value poo"})
     end
 
-    emits = d.emits
-    assert_equal 2, emits.length
-    assert_equal ["output.test", time, {"val2"=>"sed-ed value bar"}], emits[0]
-    assert_equal_event_time time, emits[0][1]
-    assert_equal ["output.test", time, {"val2"=>"sed-ed value poo"}], emits[1]
-    assert_equal_event_time time, emits[1][1]
+    assert_equal "test\t1293941655\tsed-ed value foo\n", d.formatted[0]
+    assert_equal "test\t1293941655\tsed-ed value poo\n", d.formatted[1]
+
+    events = d.events
+    assert_equal 2, events.length
+    assert_equal_event_time time, events[0][1]
+    assert_equal ["output.test", time, {"val2"=>"sed-ed value bar"}], events[0]
+    assert_equal_event_time time, events[1][1]
+    assert_equal ["output.test", time, {"val2"=>"sed-ed value poo"}], events[1]
   end
 
-  def test_json_1
-    d = create_driver(%[
-      command cat
-      in_keys message
-      out_format json
-      time_key time
+  CONFIG_JSON = %[
+    command cat
+    <format>
+      @type tsv
+      keys message
+    </format>
+    <parse>
+      @type json
+    </parse>
+    <extract>
       tag_key tag
-    ], 'input.test')
-
-    time = Fluent::EventTime.parse("2011-01-02 13:14:15")
+      time_key time
+    </extract>
+  ]
+  CONFIG_JSON_COMPAT = %[
+    command cat
+    in_keys message
+    out_format json
+    time_key time
+    tag_key tag
+  ]
 
-    d.expected_emits_length = 1
-    d.run do
-      d.emit({"message"=>%[{"time":#{time},"tag":"t1","k1":"v1"}]}, time+10)
+  data(
+    'with sections' => CONFIG_JSON,
+    'traditional' => CONFIG_JSON_COMPAT,
+  )
+  test 'using json format' do |conf|
+    d = create_driver(conf)
+    time = event_time("2011-01-02 13:14:15 +0900")
+
+    d.run(default_tag: 'input.test', expect_emits: 1, timeout: 10) do
+      i = d.instance
+      assert{ i.router }
+      d.feed(time, {"message"=>%[{"time":#{time},"tag":"t1","k1":"v1"}]})
     end
 
-    emits = d.emits
-    assert_equal 1, emits.length
-    assert_equal ["t1", time, {"k1"=>"v1"}], emits[0]
-    assert_equal_event_time time, emits[0][1]
+    assert_equal '{"time":1293941655,"tag":"t1","k1":"v1"}' + "\n", d.formatted[0]
+
+    events = d.events
+    assert_equal 1, events.length
+    assert_equal_event_time time, events[0][1]
+    assert_equal ["t1", time, {"k1"=>"v1"}], events[0]
   end
 
-  def test_json_with_float_time
-    d = create_driver(%[
-      command cat
-      in_keys message
-      out_format json
-      time_key time
+  CONFIG_JSON_WITH_FLOAT_TIME = %[
+    command cat
+    <format>
+      @type tsv
+      keys message
+    </format>
+    <parse>
+      @type json
+    </parse>
+    <extract>
       tag_key tag
-    ], 'input.test')
+      time_key time
+    </extract>
+  ]
+  CONFIG_JSON_WITH_FLOAT_TIME_COMPAT = %[
+    command cat
+    in_keys message
+    out_format json
+    time_key time
+    tag_key tag
+  ]
 
-    float_time = Time.parse("2011-01-02 13:14:15").to_f
-    time = Fluent::EventTime.from_time(Time.at(float_time))
+  data(
+    'with sections' => CONFIG_JSON_WITH_FLOAT_TIME,
+    'traditional' => CONFIG_JSON_WITH_FLOAT_TIME_COMPAT,
+  )
+  test 'using json format with float time' do |conf|
+    d = create_driver(conf)
+    time = event_time("2011-01-02 13:14:15.123 +0900")
 
-    d.expected_emits_length = 1
-    d.run do
-      d.emit({"message"=>%[{"time":#{float_time},"tag":"t1","k1":"v1"}]}, time+10)
+    d.run(default_tag: 'input.test', expect_emits: 1, timeout: 10) do
+      d.feed(time + 10, {"message"=>%[{"time":#{time.sec}.#{time.nsec},"tag":"t1","k1":"v1"}]})
     end
 
-    emits = d.emits
-    assert_equal 1, emits.length
-    assert_equal ["t1", time, {"k1"=>"v1"}], emits[0]
-    assert_equal_event_time time, emits[0][1]
+    assert_equal '{"time":1293941655.123000000,"tag":"t1","k1":"v1"}' + "\n", d.formatted[0]
+
+    events = d.events
+    assert_equal 1, events.length
+    assert_equal_event_time time, events[0][1]
+    assert_equal ["t1", time, {"k1"=>"v1"}], events[0]
   end
 
-  def test_json_with_time_format
-    d = create_driver(%[
-      command cat
-      in_keys message
-      out_format json
+  CONFIG_JSON_WITH_TIME_FORMAT = %[
+    command cat
+    <format>
+      @type tsv
+      keys message
+    </format>
+    <parse>
+      @type json
+    </parse>
+    <extract>
+      tag_key tag
       time_key time
+      time_type string
       time_format %d/%b/%Y %H:%M:%S.%N %z
-      tag_key tag
-    ], 'input.test')
+    </extract>
+  ]
+  CONFIG_JSON_WITH_TIME_FORMAT_COMPAT = %[
+    command cat
+    in_keys message
+    out_format json
+    time_key time
+    time_format %d/%b/%Y %H:%M:%S.%N %z
+    tag_key tag
+  ]
 
+  data(
+    'with sections' => CONFIG_JSON_WITH_TIME_FORMAT,
+    'traditional' => CONFIG_JSON_WITH_TIME_FORMAT_COMPAT,
+  )
+  test 'using json format with custom time format' do |conf|
+    d = create_driver(conf)
     time_str = "28/Feb/2013 12:00:00.123456789 +0900"
-    time = Fluent::EventTime.from_time(Time.strptime(time_str, "%d/%b/%Y %H:%M:%S.%N %z"))
+    time = event_time(time_str, format: "%d/%b/%Y %H:%M:%S.%N %z")
 
-    d.expected_emits_length = 1
-    d.run do
-      d.emit({"message"=>%[{"time":"#{time_str}","tag":"t1","k1":"v1"}]}, time+10)
+    d.run(default_tag: 'input.test', expect_emits: 1, timeout: 10) do
+      d.feed(time + 10, {"message"=>%[{"time":"#{time_str}","tag":"t1","k1":"v1"}]})
     end
 
-    emits = d.emits
-    assert_equal 1, emits.length
-    assert_equal ["t1", time, {"k1"=>"v1"}], emits[0]
-    assert_equal_event_time time, emits[0][1]
+    assert_equal '{"time":"28/Feb/2013 12:00:00.123456789 +0900","tag":"t1","k1":"v1"}' + "\n", d.formatted[0]
+
+    events = d.events
+    assert_equal 1, events.length
+    assert_equal_event_time time, events[0][1]
+    assert_equal ["t1", time, {"k1"=>"v1"}], events[0]
   end
-end
 
+  CONFIG_ROUND_ROBIN = %[
+    command ruby -e 'STDOUT.sync = true; STDIN.each_line{|line| puts line.chomp + "\t" + Process.pid.to_s }'
+    num_children 3
+    <inject>
+      tag_key     tag
+      time_key    time_in
+      time_type   string
+      time_format %Y-%m-%d %H:%M:%S
+    </inject>
+    <format>
+      keys ["time_in", "tag", "k1"]
+    </format>
+    <parse>
+      keys ["time_out", "tag", "k2", "child_pid"]
+    </parse>
+    <extract>
+      tag_key     tag
+      time_key    time_out
+      time_type   string
+      time_format %Y-%m-%d %H:%M:%S
+    </extract>
+  ]
+  CONFIG_ROUND_ROBIN_COMPAT = %[
+    command ruby -e 'STDOUT.sync = true; STDIN.each_line{|line| puts line.chomp + "\t" + Process.pid.to_s }'
+    in_keys time_in,tag,k1
+    out_keys time_out,tag,k2,child_pid
+    tag_key tag
+    in_time_key time_in
+    out_time_key time_out
+    time_format %Y-%m-%d %H:%M:%S
+    localtime
+    num_children 3
+  ]
+
+  data(
+    'with sections' => CONFIG_ROUND_ROBIN,
+    'traditional' => CONFIG_ROUND_ROBIN_COMPAT,
+  )
+  test 'using child processes by round robin' do |conf|
+    d = create_driver(conf)
+    time = event_time("2011-01-02 13:14:15")
+
+    d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: true,  shutdown: false){ d.feed(time, {"k1"=>1}) }
+    d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1"=>2}) }
+    d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1"=>3}) }
+    d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1"=>4}) }
+    d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1"=>5}) }
+    d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1"=>6}) }
+    d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1"=>7}) }
+    d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1"=>8}) }
+    d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: true ){ d.feed(time, {"k1"=>9}) }
+
+    assert_equal "2011-01-02 13:14:15\ttest\t1\n", d.formatted[0]
+    assert_equal "2011-01-02 13:14:15\ttest\t2\n", d.formatted[1]
+    assert_equal "2011-01-02 13:14:15\ttest\t3\n", d.formatted[2]
+    assert_equal "2011-01-02 13:14:15\ttest\t4\n", d.formatted[3]
+    assert_equal "2011-01-02 13:14:15\ttest\t5\n", d.formatted[4]
+    assert_equal "2011-01-02 13:14:15\ttest\t6\n", d.formatted[5]
+    assert_equal "2011-01-02 13:14:15\ttest\t7\n", d.formatted[6]
+    assert_equal "2011-01-02 13:14:15\ttest\t8\n", d.formatted[7]
+    assert_equal "2011-01-02 13:14:15\ttest\t9\n", d.formatted[8]
+
+    events = d.events
+    assert_equal 9, events.length
+
+    pid_list = []
+    events.each do |event|
+      pid = event[2]['child_pid']
+      pid_list << pid unless pid_list.include?(pid)
+    end
+    assert_equal 3, pid_list.size, "the number of pids should be same with number of child processes: #{pid_list.inspect}"
+
+    assert_equal pid_list[0], events[0][2]['child_pid']
+    assert_equal pid_list[1], events[1][2]['child_pid']
+    assert_equal pid_list[2], events[2][2]['child_pid']
+    assert_equal pid_list[0], events[3][2]['child_pid']
+    assert_equal pid_list[1], events[4][2]['child_pid']
+    assert_equal pid_list[2], events[5][2]['child_pid']
+    assert_equal pid_list[0], events[6][2]['child_pid']
+    assert_equal pid_list[1], events[7][2]['child_pid']
+    assert_equal pid_list[2], events[8][2]['child_pid']
+  end
+
+  # child process exits per 3 lines
+  CONFIG_RESPAWN = %[
+    command ruby -e 'STDOUT.sync = true; proc = ->(){line = STDIN.readline.chomp; puts line + "\t" + Process.pid.to_s}; proc.call; proc.call; proc.call'
+    num_children 4
+    child_respawn -1
+    <inject>
+      tag_key   tag
+      time_key  time_in
+      time_type unixtime
+    </inject>
+    <format>
+      keys ["time_in", "tag", "k1"]
+    </format>
+    <parse>
+      keys ["time_out", "tag", "k2", "child_pid"]
+    </parse>
+    <extract>
+      tag_key   tag
+      time_key  time_out
+      time_type unixtime
+    </extract>
+  ]
+
+  CONFIG_RESPAWN_COMPAT = %[
+    command ruby -e 'STDOUT.sync = true; proc = ->(){line = STDIN.readline.chomp; puts line + "\t" + Process.pid.to_s}; proc.call; proc.call; proc.call'
+    num_children 4
+    child_respawn -1
+    in_keys time_in,tag,k1
+    out_keys time_out,tag,k2,child_pid
+    tag_key tag
+    in_time_key time_in
+    out_time_key time_out
+#    time_format %Y-%m-%d %H:%M:%S
+#    localtime
+  ]
+
+  data(
+    'with sections' => CONFIG_RESPAWN,
+    'traditional' => CONFIG_RESPAWN_COMPAT,
+  )
+  test 'emit events via child processes which exits sometimes' do |conf|
+    d = create_driver(conf)
+    time = event_time("2011-01-02 13:14:15")
+
+    countup = 0
+
+    d.run(start: true, shutdown: false)
+
+    assert_equal 4, d.instance.instance_eval{ @_child_process_processes.size }
+
+    20.times do
+      d.run(default_tag: 'test', expect_emits: 1, timeout: 10, force_flush_retry: true, start: false, shutdown: false) do
+        d.feed(time, {"k1"=>countup}); countup += 1
+        d.feed(time, {"k1"=>countup}); countup += 1
+        d.feed(time, {"k1"=>countup}); countup += 1
+      end
+    end
+
+    events = d.events
+    assert_equal 60, events.length
+
+    pid_list = []
+    events.each do |event|
+      pid = event[2]['child_pid']
+      pid_list << pid unless pid_list.include?(pid)
+    end
+    # the number of pids should be same with number of child processes
+    assert{ pid_list.size >= 18 }
+
+    logs = d.instance.log.out.logs
+    assert{ logs.select{|l| l.include?("child process exits with error code") }.size >= 18 } # 20
+    assert{ logs.select{|l| l.include?("respawning child process") }.size >= 18 } # 20
+
+    d.run(start: false, shutdown: true)
+  end
+end