fluentd 0.14.4-x86-mingw32

data/test/plugin/test_formatter_single_value.rb

@@ -0,0 +1,38 @@
+require_relative '../helper'
+require 'fluent/test/driver/formatter'
+require 'fluent/plugin/formatter_single_value'
+
+class SingleValueFormatterTest < ::Test::Unit::TestCase
+  def create_driver(conf = "")
+    Fluent::Test::Driver::Formatter.new(Fluent::Plugin::SingleValueFormatter).configure(conf)
+  end
+
+  def test_config_params
+    d = create_driver
+    assert_equal "message", d.instance.message_key
+  end
+
+  def test_config_params_message_key
+    d = create_driver('message_key' => 'foobar')
+    assert_equal "foobar", d.instance.message_key
+  end
+
+  def test_format
+    d = create_driver
+    formatted = d.instance.format('tag', event_time, {'message' => 'awesome'})
+    assert_equal("awesome\n", formatted)
+  end
+
+  def test_format_without_newline
+    d = create_driver('add_newline' => 'false')
+    formatted = d.instance.format('tag', event_time, {'message' => 'awesome'})
+    assert_equal("awesome", formatted)
+  end
+
+  def test_format_with_message_key
+    d = create_driver('message_key' => 'foobar')
+    formatted = d.instance.format('tag', event_time, {'foobar' => 'foo'})
+
+    assert_equal("foo\n", formatted)
+  end
+end

data/test/plugin/test_in_debug_agent.rb

@@ -0,0 +1,28 @@
+require_relative '../helper'
+require 'fluent/test/driver/input'
+require 'fluent/plugin/in_debug_agent'
+require 'fileutils'
+
+class DebugAgentInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    FileUtils.rm_rf(TMP_DIR)
+    FileUtils.mkdir_p(TMP_DIR)
+  end
+
+  TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/in_debug_agent")
+
+  def create_driver(conf = '')
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::DebugAgentInput).configure(conf)
+  end
+
+  def test_unix_path_writable
+    assert_nothing_raised do
+      create_driver %[unix_path #{TMP_DIR}/test_path]
+    end
+
+    assert_raise(Fluent::ConfigError) do
+      create_driver %[unix_path #{TMP_DIR}/does_not_exist/test_path]
+    end
+  end
+end

data/test/plugin/test_in_dummy.rb

@@ -0,0 +1,188 @@
+require_relative '../helper'
+require 'fluent/test/driver/input'
+require 'fluent/plugin/in_dummy'
+require 'fileutils'
+
+class DummyTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::DummyInput).configure(conf)
+  end
+
+  sub_test_case 'configure' do
+    test 'required parameters' do
+      assert_raise_message("'tag' parameter is required") do
+        create_driver('')
+      end
+    end
+
+    test 'tag' do
+      d = create_driver(%[
+        tag dummy
+      ])
+      assert_equal "dummy", d.instance.tag
+    end
+
+    config = %[
+      tag dummy
+    ]
+
+    test 'auto_increment_key' do
+      d = create_driver(config + %[
+        auto_increment_key id
+      ])
+      assert_equal "id", d.instance.auto_increment_key
+    end
+
+    test 'rate' do
+      d = create_driver(config + %[
+        rate 10
+      ])
+      assert_equal 10, d.instance.rate
+    end
+
+    test 'dummy' do
+      # hash is okay
+      d = create_driver(config + %[dummy {"foo":"bar"}])
+      assert_equal [{"foo"=>"bar"}], d.instance.dummy
+
+      # array of hash is okay
+      d = create_driver(config + %[dummy [{"foo":"bar"}]])
+      assert_equal [{"foo"=>"bar"}], d.instance.dummy
+
+      assert_raise_message(/JSON::ParserError|got incomplete JSON/) do
+        create_driver(config + %[dummy "foo"])
+      end
+
+      assert_raise_message(/is not a hash/) do
+        create_driver(config + %[dummy ["foo"]])
+      end
+    end
+  end
+
+  sub_test_case "emit" do
+    config = %[
+      tag dummy
+      rate 10
+      dummy {"foo":"bar"}
+    ]
+
+    test 'simple' do
+      d = create_driver(config)
+      d.run(timeout: 0.5)
+
+      d.events.each do |tag, time, record|
+        assert_equal("dummy", tag)
+        assert_equal({"foo"=>"bar"}, record)
+        assert(time.is_a?(Fluent::EventTime))
+      end
+    end
+
+    test 'with auto_increment_key' do
+      d = create_driver(config + %[auto_increment_key id])
+      d.run(timeout: 0.5)
+
+      d.events.each_with_index do |(tag, _time, record), i|
+        assert_equal("dummy", tag)
+        assert_equal({"foo"=>"bar", "id"=>i}, record)
+      end
+    end
+  end
+
+  TEST_PLUGIN_STORAGE_PATH = File.join( File.dirname(File.dirname(__FILE__)), 'tmp', 'in_dummy', 'store' )
+  FileUtils.mkdir_p TEST_PLUGIN_STORAGE_PATH
+
+  sub_test_case "doesn't suspend internal counters in default" do
+    config1 = {
+      'tag' => 'dummy',
+      'rate' => '2',
+      'dummy' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
+      'auto_increment_key' => 'id',
+      'suspend' => false,
+    }
+    conf1 = config_element('ROOT', '', config1, [])
+    test "value of auto increment key is not suspended after stop-and-start" do
+      assert !File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-01.json'))
+
+      d1 = create_driver(conf1)
+      d1.run(timeout: 0.5) do
+        d1.instance.emit(4)
+      end
+
+      first_id1 = d1.events.first[2]['id']
+      assert_equal 0, first_id1
+
+      last_id1 = d1.events.last[2]['id']
+      assert { last_id1 > 0 }
+
+      assert !File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-01.json'))
+
+      d2 = create_driver(conf1)
+      d2.run(timeout: 0.5) do
+        d2.instance.emit(4)
+      end
+
+      first_id2 = d2.events.first[2]['id']
+      assert_equal 0, first_id2
+
+      assert !File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-01.json'))
+    end
+  end
+
+  sub_test_case "suspend internal counters if suspend is true" do
+    setup do
+      FileUtils.rm_rf(TEST_PLUGIN_STORAGE_PATH)
+      FileUtils.mkdir_p(File.join(TEST_PLUGIN_STORAGE_PATH, 'json'))
+      FileUtils.chmod_R(0755, File.join(TEST_PLUGIN_STORAGE_PATH, 'json'))
+    end
+
+    config2 = {
+      '@id' => 'test-02',
+      'tag' => 'dummy',
+      'rate' => '2',
+      'dummy' => '[{"x": 1, "y": "1"}, {"x": 2, "y": "2"}, {"x": 3, "y": "3"}]',
+      'auto_increment_key' => 'id',
+      'suspend' => true,
+    }
+    conf2 = config_element('ROOT', '', config2, [
+              config_element(
+                'storage', '',
+                {'@type' => 'local',
+                 '@id' => 'test-02',
+                 'path' => File.join(TEST_PLUGIN_STORAGE_PATH,
+                                     'json', 'test-02.json'),
+                 'persistent' => true,
+                })
+            ])
+    test "value of auto increment key is suspended after stop-and-start" do
+      assert !File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-02.json'))
+
+      d1 = create_driver(conf2)
+      d1.run(timeout: 0.5) do
+        d1.instance.emit(4)
+      end
+
+      first_id1 = d1.events.first[2]['id']
+      assert_equal 0, first_id1
+
+      last_id1 = d1.events.last[2]['id']
+      assert { last_id1 > 0 }
+
+      assert File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-02.json'))
+
+      d2 = create_driver(conf2)
+      d2.run(timeout: 0.5) do
+        d2.instance.emit(4)
+      end
+      d2.events
+
+      first_id2 = d2.events.first[2]['id']
+      assert_equal last_id1 + 1, first_id2
+
+      assert File.exist?(File.join(TEST_PLUGIN_STORAGE_PATH, 'json', 'test-02.json'))
+    end
+  end
+end

data/test/plugin/test_in_exec.rb

@@ -0,0 +1,133 @@
+require_relative '../helper'
+require 'fluent/test/driver/input'
+require 'fluent/plugin/in_exec'
+require 'net/http'
+
+class ExecInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+    @test_time = event_time("2011-01-02 13:14:15")
+    @script = File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts', 'exec_script.rb'))
+  end
+
+  def create_driver(conf = tsv_config)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::ExecInput).configure(conf)
+  end
+
+  def tsv_config
+    %[
+      command ruby #{@script} "2011-01-02 13:14:15" 0
+      keys time,tag,k1
+      time_key time
+      tag_key tag
+      time_format %Y-%m-%d %H:%M:%S
+      run_interval 1s
+    ]
+  end
+
+  def json_config
+    %[
+      command ruby #{@script} #{@test_time} 1
+      format json
+      tag_key tag
+      time_key time
+      run_interval 1s
+    ]
+  end
+
+  def msgpack_config
+    %[
+      command ruby #{@script} #{@test_time} 2
+      format msgpack
+      tag_key tagger
+      time_key datetime
+      run_interval 1s
+    ]
+  end
+
+  def regexp_config
+    %[
+      command ruby #{@script} "2011-01-02 13:14:15" 3
+      format /(?<time>[^\\\]]*) (?<message>[^ ]*)/
+      tag regex_tag
+      run_interval 1s
+    ]
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal 'tsv', d.instance.format
+    assert_equal ["time","tag","k1"], d.instance.keys
+    assert_equal "tag", d.instance.tag_key
+    assert_equal "time", d.instance.time_key
+    assert_equal "%Y-%m-%d %H:%M:%S", d.instance.time_format
+  end
+
+  def test_configure_with_json
+    d = create_driver json_config
+    assert_equal 'json', d.instance.format
+    assert_equal [], d.instance.keys
+  end
+
+  def test_configure_with_msgpack
+    d = create_driver msgpack_config
+    assert_equal 'msgpack', d.instance.format
+    assert_equal [], d.instance.keys
+  end
+
+  def test_configure_with_regexp
+    d = create_driver regexp_config
+    assert_equal '/(?<time>[^\]]*) (?<message>[^ ]*)/', d.instance.format
+    assert_equal 'regex_tag', d.instance.tag
+  end
+
+  # TODO: Merge following tests into one case with parameters
+
+  def test_emit
+    d = create_driver
+
+    d.run(expect_emits: 2)
+
+    assert_equal true, d.events.length > 0
+    d.events.each_with_index {|event, i|
+      assert_equal ["tag1", @test_time, {"k1"=>"ok"}], event
+      assert_equal_event_time(@test_time, event[1])
+    }
+  end
+
+  def test_emit_json
+    d = create_driver json_config
+
+    d.run(expect_emits: 2)
+
+    assert_equal true, d.events.length > 0
+    d.events.each_with_index {|event, i|
+      assert_equal ["tag1", @test_time, {"k1"=>"ok"}], event
+      assert_equal_event_time(@test_time, event[1])
+    }
+  end
+
+  def test_emit_msgpack
+    d = create_driver msgpack_config
+
+    d.run(expect_emits: 2)
+
+    assert_equal true, d.events.length > 0
+    d.events.each_with_index {|event, i|
+      assert_equal ["tag1", @test_time, {"k1"=>"ok"}], event
+      assert_equal_event_time(@test_time, event[1])
+    }
+  end
+
+  def test_emit_regexp
+    d = create_driver regexp_config
+
+    d.run(expect_emits: 2)
+
+    assert_equal true, d.events.length > 0
+    d.events.each_with_index {|event, i|
+      assert_equal ["regex_tag", @test_time, {"message"=>"hello"}], event
+      assert_equal_event_time(@test_time, event[1])
+    }
+  end
+end

data/test/plugin/test_in_forward.rb

@@ -0,0 +1,635 @@
+require_relative '../helper'
+
+require 'fluent/test'
+require 'base64'
+
+require 'fluent/env'
+require 'fluent/plugin/in_forward'
+
+class ForwardInputTest < Test::Unit::TestCase
+  class << self
+    def startup
+      socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+      @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
+      ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+    end
+
+    def shutdown
+      @server.close
+    end
+  end
+
+  def setup
+    Fluent::Test.setup
+    @responses = []  # for testing responses after sending data
+  end
+
+  PORT = unused_port
+  CONFIG = %[
+    port #{PORT}
+    bind 127.0.0.1
+  ]
+  PEERADDR = ['?', '0000', '127.0.0.1', '127.0.0.1']
+
+  def create_driver(conf=CONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::ForwardInput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal PORT, d.instance.port
+    assert_equal '127.0.0.1', d.instance.bind
+    assert_equal 0, d.instance.linger_timeout
+    assert_equal 0.5, d.instance.blocking_timeout
+    assert !d.instance.backlog
+  end
+
+  # TODO: Will add Loop::run arity check with stub/mock library
+
+  def connect
+    TCPSocket.new('127.0.0.1', PORT)
+  end
+
+  def test_time
+    d = create_driver
+
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+    Fluent::Engine.now = time
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+
+    d.run do
+      d.expected_emits.each {|tag, _time, record|
+        send_data Fluent::Engine.msgpack_factory.packer.write([tag, 0, record]).to_s
+      }
+    end
+  end
+
+  def test_message
+    d = create_driver
+
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+
+    d.run do
+      d.expected_emits.each {|tag, _time, record|
+        send_data Fluent::Engine.msgpack_factory.packer.write([tag, _time, record]).to_s
+      }
+    end
+  end
+
+  def test_message_with_time_as_integer
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+
+    d.run do
+      d.expected_emits.each {|tag, _time, record|
+        send_data Fluent::Engine.msgpack_factory.packer.write([tag, _time, record]).to_s
+      }
+    end
+  end
+
+  def test_message_with_skip_invalid_event
+    d = create_driver(CONFIG + "skip_invalid_event true")
+
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+    d.expect_emit "tag1", time, {"a" => 1}
+    d.expect_emit "tag2", time, {"a" => 2}
+
+    d.run do
+      entries = d.expected_emits.map {|tag, _time, record| [tag, _time, record] }
+      # These entries are skipped
+      entries << ['tag1', true, {'a' => 3}] << ['tag2', time, 'invalid record']
+
+      entries.each {|tag, _time, record|
+        # Without ack, logs are sometimes not saved to logs during test.
+        send_data Fluent::Engine.msgpack_factory.packer.write([tag, _time, record]).to_s, true
+      }
+    end
+
+    assert_equal 2, d.instance.log.logs.count { |line| line =~ /got invalid event and drop it/ }
+  end
+
+  def test_forward
+    d = create_driver
+
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag1", time, {"a"=>2}
+
+    d.run do
+      entries = []
+      d.expected_emits.each {|tag, _time,record|
+        entries << [time, record]
+      }
+      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
+    end
+  end
+
+  def test_forward_with_time_as_integer
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag1", time, {"a"=>2}
+
+    d.run do
+      entries = []
+      d.expected_emits.each {|_tag, _time, record|
+        entries << [_time, record]
+      }
+      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
+    end
+  end
+
+  def test_forward_with_skip_invalid_event
+    d = create_driver(CONFIG + "skip_invalid_event true")
+
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+    d.expect_emit "tag1", time, {"a" => 1}
+    d.expect_emit "tag1", time, {"a" => 2}
+
+    d.run do
+      entries = d.expected_emits.map {|_tag, _time, record| [_time, record] }
+      # These entries are skipped
+      entries << ['invalid time', {'a' => 3}] << [time, 'invalid record']
+
+      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
+    end
+
+    assert_equal 2, d.instance.log.logs.count { |line| line =~ /skip invalid event/ }
+  end
+
+  def test_packed_forward
+    d = create_driver
+
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag1", time, {"a"=>2}
+
+    d.run do
+      entries = ''
+      d.expected_emits.each {|_tag, _time, record|
+        Fluent::Engine.msgpack_factory.packer(entries).write([_time, record]).flush
+      }
+      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
+    end
+  end
+
+  def test_packed_forward_with_time_as_integer
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag1", time, {"a"=>2}
+
+    d.run do
+      entries = ''
+      d.expected_emits.each {|_tag, _time, record|
+        Fluent::Engine.msgpack_factory.packer(entries).write([_time, record]).flush
+      }
+      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
+    end
+  end
+
+  def test_packed_forward_with_skip_invalid_event
+    d = create_driver(CONFIG + "skip_invalid_event true")
+
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+    d.expect_emit "tag1", time, {"a" => 1}
+    d.expect_emit "tag1", time, {"a" => 2}
+
+    d.run do
+      entries = d.expected_emits.map {|_tag , _time, record| [_time, record] }
+      # These entries are skipped
+      entries << ['invalid time', {'a' => 3}] << [time, 'invalid record']
+
+      packed_entries = ''
+      entries.each {|_time, record|
+        Fluent::Engine.msgpack_factory.packer(packed_entries).write([_time, record]).flush
+      }
+      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", packed_entries]).to_s
+    end
+
+    assert_equal 2, d.instance.log.logs.count { |line| line =~ /skip invalid event/ }
+  end
+
+  def test_message_json
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>2}
+
+    d.run do
+      d.expected_emits.each {|tag, _time, record|
+        send_data [tag, _time, record].to_json
+      }
+    end
+  end
+
+  def test_set_size_to_option
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag1", time, {"a"=>2}]
+    ]
+
+    entries = ''
+    events.each {|_tag, _time, record|
+      [_time, record].to_msgpack(entries)
+    }
+
+    chunk = ["tag1", entries, { 'size' => events.length }].to_msgpack
+
+    d.run do
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
+        option = d.instance.send(:on_message, obj, chunk.size, PEERADDR)
+        assert_equal option['size'], events.length
+      end
+    end
+  end
+
+  def test_send_large_chunk_warning
+    d = create_driver(CONFIG + %[
+      chunk_size_warn_limit 16M
+      chunk_size_limit 32M
+    ])
+
+    time = Fluent::EventTime.parse("2014-04-25 13:14:15 UTC")
+
+    # generate over 16M chunk
+    str = "X" * 1024 * 1024
+    chunk = [ "test.tag", (0...16).map{|i| [time + i, {"data" => str}] } ].to_msgpack
+    assert chunk.size > (16 * 1024 * 1024)
+    assert chunk.size < (32 * 1024 * 1024)
+
+    d.run do
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
+        d.instance.send(:on_message, obj, chunk.size, PEERADDR)
+      end
+    end
+
+    # check emitted data
+    emits = d.emits
+    assert_equal 16, emits.size
+    assert emits.map(&:first).all?{|t| t == "test.tag" }
+    assert_equal (0...16).to_a, emits.map{|_tag, t, _record| t - time }
+
+    # check log
+    assert d.instance.log.logs.select{|line|
+      line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_warn_limit':/ &&
+      line =~ / tag="test.tag" source="host: 127.0.0.1, addr: 127.0.0.1, port: \d+" limit=16777216 size=16777501/
+    }.size == 1, "large chunk warning is not logged"
+  end
+
+  def test_send_large_chunk_only_warning
+    d = create_driver(CONFIG + %[
+      chunk_size_warn_limit 16M
+    ])
+    time = Fluent::EventTime.parse("2014-04-25 13:14:15 UTC")
+
+    # generate over 16M chunk
+    str = "X" * 1024 * 1024
+    chunk = [ "test.tag", (0...16).map{|i| [time + i, {"data" => str}] } ].to_msgpack
+
+    d.run do
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
+        d.instance.send(:on_message, obj, chunk.size, PEERADDR)
+      end
+    end
+
+    # check log
+    assert d.instance.log.logs.select{ |line|
+      line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_warn_limit':/ &&
+      line =~ / tag="test.tag" source="host: 127.0.0.1, addr: 127.0.0.1, port: \d+" limit=16777216 size=16777501/
+    }.size == 1, "large chunk warning is not logged"
+  end
+
+  def test_send_large_chunk_limit
+    d = create_driver(CONFIG + %[
+      chunk_size_warn_limit 16M
+      chunk_size_limit 32M
+    ])
+
+    time = Fluent::EventTime.parse("2014-04-25 13:14:15 UTC")
+
+    # generate over 32M chunk
+    str = "X" * 1024 * 1024
+    chunk = [ "test.tag", (0...32).map{|i| [time + i, {"data" => str}] } ].to_msgpack
+    assert chunk.size > (32 * 1024 * 1024)
+
+    # d.run => send_data
+    d.run do
+      Fluent::Engine.msgpack_factory.unpacker.feed_each(chunk) do |obj|
+        d.instance.send(:on_message, obj, chunk.size, PEERADDR)
+      end
+    end
+
+    # check emitted data
+    emits = d.emits
+    assert_equal 0, emits.size
+
+    # check log
+    assert d.instance.log.logs.select{|line|
+      line =~ / \[warn\]: Input chunk size is larger than 'chunk_size_limit', dropped:/ &&
+      line =~ / tag="test.tag" source="host: 127.0.0.1, addr: 127.0.0.1, port: \d+" limit=33554432 size=33554989/
+    }.size == 1, "large chunk warning is not logged"
+  end
+
+  data('string chunk' => 'broken string',
+       'integer chunk' => 10)
+  def test_send_broken_chunk(data)
+    d = create_driver
+
+    # d.run => send_data
+    d.run do
+      d.instance.send(:on_message, data, 1000000000, PEERADDR)
+    end
+
+    # check emitted data
+    emits = d.emits
+    assert_equal 0, emits.size
+
+    # check log
+    assert d.instance.log.logs.select{|line|
+      line =~ / \[warn\]: incoming chunk is broken: source="host: 127.0.0.1, addr: 127.0.0.1, port: \d+" msg=#{data.inspect}/
+    }.size == 1, "should not accept broken chunk"
+  end
+
+  def test_respond_to_message_requiring_ack
+    d = create_driver
+
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag2", time, {"a"=>2}]
+    ]
+    d.expected_emits_length = events.length
+
+    expected_acks = []
+
+    d.run do
+      events.each {|tag, _time, record|
+        op = { 'chunk' => Base64.encode64(record.object_id.to_s) }
+        expected_acks << op['chunk']
+        send_data [tag, _time, record, op].to_msgpack, true
+      }
+    end
+
+    assert_equal events, d.emits
+    assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }
+  end
+
+  # FIX: response is not pushed into @responses because IO.select has been blocked until InputForward shutdowns
+  def test_respond_to_forward_requiring_ack
+    d = create_driver
+
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag1", time, {"a"=>2}]
+    ]
+    d.expected_emits_length = events.length
+
+    expected_acks = []
+
+    d.run do
+      entries = []
+      events.each {|_tag, _time, record|
+        entries << [time, record]
+      }
+      op = { 'chunk' => Base64.encode64(entries.object_id.to_s) }
+      expected_acks << op['chunk']
+      send_data ["tag1", entries, op].to_msgpack, true
+    end
+
+    assert_equal events, d.emits
+    assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }
+  end
+
+  def test_respond_to_packed_forward_requiring_ack
+    d = create_driver
+
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag1", time, {"a"=>2}]
+    ]
+    d.expected_emits_length = events.length
+
+    expected_acks = []
+
+    d.run do
+      entries = ''
+      events.each {|_tag, _time, record|
+        [_time, record].to_msgpack(entries)
+      }
+      op = { 'chunk' => Base64.encode64(entries.object_id.to_s) }
+      expected_acks << op['chunk']
+      send_data ["tag1", entries, op].to_msgpack, true
+    end
+
+    assert_equal events, d.emits
+    assert_equal expected_acks, @responses.map { |res| MessagePack.unpack(res)['ack'] }
+  end
+
+  def test_respond_to_message_json_requiring_ack
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag2", time, {"a"=>2}]
+    ]
+    d.expected_emits_length = events.length
+
+    expected_acks = []
+
+    d.run do
+      events.each {|tag, _time, record|
+        op = { 'chunk' => Base64.encode64(record.object_id.to_s) }
+        expected_acks << op['chunk']
+        send_data [tag, _time, record, op].to_json, true
+      }
+    end
+
+    assert_equal events, d.emits
+    assert_equal expected_acks, @responses.map { |res| JSON.parse(res)['ack'] }
+  end
+
+  def test_not_respond_to_message_not_requiring_ack
+    d = create_driver
+
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag2", time, {"a"=>2}]
+    ]
+    d.expected_emits_length = events.length
+
+    d.run do
+      events.each {|tag, _time, record|
+        send_data [tag, _time, record].to_msgpack, true
+      }
+    end
+
+    assert_equal events, d.emits
+    assert_equal [nil, nil], @responses
+  end
+
+  def test_not_respond_to_forward_not_requiring_ack
+    d = create_driver
+
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag1", time, {"a"=>2}]
+    ]
+    d.expected_emits_length = events.length
+
+    d.run do
+      entries = []
+      events.each {|_tag, _time, record|
+        entries << [_time, record]
+      }
+      send_data ["tag1", entries].to_msgpack, true
+    end
+
+    assert_equal events, d.emits
+    assert_equal [nil], @responses
+  end
+
+  def test_not_respond_to_packed_forward_not_requiring_ack
+    d = create_driver
+
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag1", time, {"a"=>2}]
+    ]
+    d.expected_emits_length = events.length
+
+    d.run do
+      entries = ''
+      events.each {|_tag, _time, record|
+        [_time, record].to_msgpack(entries)
+      }
+      send_data ["tag1", entries].to_msgpack, true
+    end
+
+    assert_equal events, d.emits
+    assert_equal [nil], @responses
+  end
+
+  def test_not_respond_to_message_json_not_requiring_ack
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag2", time, {"a"=>2}]
+    ]
+    d.expected_emits_length = events.length
+
+    d.run do
+      events.each {|tag, _time, record|
+        send_data [tag, _time, record].to_json, true
+      }
+    end
+
+    assert_equal events, d.emits
+    assert_equal [nil, nil], @responses
+  end
+
+  def send_data(data, try_to_receive_response=false, response_timeout=1)
+    io = connect
+    begin
+      io.write data
+      if try_to_receive_response
+        if IO.select([io], nil, nil, response_timeout)
+          res = io.recv(1024)
+        end
+        # timeout means no response, so push nil to @responses
+      end
+    ensure
+      io.close
+    end
+    @responses << res if try_to_receive_response
+  end
+
+  # TODO: Use sub_test_case. Currently Errno::EADDRINUSE happens inside sub_test_case
+  test 'message protocol with source_hostname_key' do
+    execute_test { |events|
+      events.each { |tag, time, record|
+        send_data [tag, time, record].to_msgpack
+      }
+    }
+  end
+
+  test 'forward protocol with source_hostname_key' do
+    execute_test { |events|
+      entries = []
+      events.each {|tag,time,record|
+        entries << [time, record]
+      }
+      send_data ['tag1', entries].to_msgpack
+    }
+  end
+
+  test 'packed forward protocol with source_hostname_key' do
+    execute_test { |events|
+      entries = ''
+      events.each { |tag, time, record|
+        Fluent::Engine.msgpack_factory.packer(entries).write([time, record]).flush
+      }
+      send_data Fluent::Engine.msgpack_factory.packer.write(["tag1", entries]).to_s
+    }
+  end
+
+  def execute_test(&block)
+    d = create_driver(CONFIG + 'source_hostname_key source')
+
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+    events = [
+      ["tag1", time, {"a"=>1}],
+      ["tag1", time, {"a"=>2}]
+    ]
+    d.expected_emits_length = events.length
+
+    d.run do
+      block.call(events)
+    end
+
+    d.emits.each { |tag, _time, record|
+      assert_true record.has_key?('source')
+    }
+  end
+
+  # TODO heartbeat
+end