fluent-plugin-elasticsearch 1.18.2 → 2.0.0.rc.1

data/test/plugin/test_out_elasticsearch.rb

@@ -1,6 +1,6 @@
 require 'helper'
 require 'date'
-require 'json'
+require 'fluent/test/driver/output'
 require 'flexmock/test_unit'
 
 class ElasticsearchOutput < Test::Unit::TestCase
@@ -16,8 +16,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
     log.out.logs.slice!(0, log.out.logs.length)
   end
 
-  def driver(tag='test', conf='')
-    @driver ||= Fluent::Test::BufferedOutputTestDriver.new(Fluent::ElasticsearchOutput, tag) {
+  def driver(conf='')
+    @driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput) {
       # v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
       if !defined?(Fluent::Plugin::Output)
         def format(tag, time, record)
@@ -27,8 +27,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
     }.configure(conf)
   end
 
-  def sample_record(content={})
-      {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}.merge(content)
+  def sample_record
+    {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}
   end
 
   def stub_elastic_ping(url="http://localhost:9200")
@@ -37,7 +37,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def stub_elastic(url="http://localhost:9200/_bulk")
     stub_request(:post, url).with do |req|
-        @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
+      @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
     end
   end
 
@@ -57,112 +57,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
     end
   end
 
-  def make_response_body(req, error_el = nil, error_status = nil, error = nil)
-    req_index_cmds = req.body.split("\n").map { |r| JSON.parse(r) }
-    items = []
-    count = 0
-    ids = 1
-    op = nil
-    index = nil
-    type = nil
-    id = nil
-    req_index_cmds.each do |cmd|
-      if count.even?
-        op = cmd.keys[0]
-        index = cmd[op]['_index']
-        type = cmd[op]['_type']
-        if cmd[op].has_key?('_id')
-          id = cmd[op]['_id']
-        else
-          # Note: this appears to be an undocumented feature of Elasticsearch
-          # https://www.elastic.co/guide/en/elasticsearch/reference/2.4/docs-bulk.html
-          # When you submit an "index" write_operation, with no "_id" field in the
-          # metadata header, Elasticsearch will turn this into a "create"
-          # operation in the response.
-          if "index" == op
-            op = "create"
-          end
-          id = ids
-          ids += 1
-        end
-      else
-        item = {
-          op => {
-            '_index' => index, '_type' => type, '_id' => id, '_version' => 1,
-            '_shards' => { 'total' => 1, 'successful' => 1, 'failed' => 0 },
-            'status' => op == 'create' ? 201 : 200
-          }
-        }
-        items.push(item)
-      end
-      count += 1
-    end
-    if !error_el.nil? && !error_status.nil? && !error.nil?
-      op = items[error_el].keys[0]
-      items[error_el][op].delete('_version')
-      items[error_el][op].delete('_shards')
-      items[error_el][op]['error'] = error
-      items[error_el][op]['status'] = error_status
-      errors = true
-    else
-      errors = false
-    end
-    @index_cmds = items
-    body = { 'took' => 6, 'errors' => errors, 'items' => items }
-    return body.to_json
-  end
-
-  def stub_elastic_bad_argument(url="http://localhost:9200/_bulk")
-    error = {
-      "type" => "mapper_parsing_exception",
-      "reason" => "failed to parse [...]",
-      "caused_by" => {
-        "type" => "illegal_argument_exception",
-        "reason" => "Invalid format: \"...\""
-      }
-    }
-    stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 400, error), :headers => { 'Content-Type' => 'json' } } })
-  end
-
-  def stub_elastic_bulk_error(url="http://localhost:9200/_bulk")
-    error = {
-      "type" => "some-unrecognized-error",
-      "reason" => "some message printed here ...",
-    }
-    stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 500, error), :headers => { 'Content-Type' => 'json' } } })
-  end
-
-  def stub_elastic_bulk_rejected(url="http://localhost:9200/_bulk")
-    error = {
-      "status" => 500,
-      "type" => "es_rejected_execution_exception",
-      "reason" => "rejected execution of org.elasticsearch.transport.TransportService$4@1a34d37a on EsThreadPoolExecutor[bulk, queue capacity = 50, org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor@312a2162[Running, pool size = 32, active threads = 32, queued tasks = 50, completed tasks = 327053]]"
-    }
-    stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 429, error), :headers => { 'Content-Type' => 'json' } } })
-  end
-
-  def stub_elastic_out_of_memory(url="http://localhost:9200/_bulk")
-    error = {
-      "status" => 500,
-      "type" => "out_of_memory_error",
-      "reason" => "Java heap space"
-    }
-    stub_request(:post, url).to_return(lambda { |req| { :status => 200, :body => make_response_body(req, 1, 500, error), :headers => { 'Content-Type' => 'json' } } })
-  end
-
-  def stub_elastic_unexpected_response_op(url="http://localhost:9200/_bulk")
-    error = {
-      "category" => "some-other-type",
-      "reason" => "some-other-reason"
-    }
-    stub_request(:post, url).to_return(lambda { |req| bodystr = make_response_body(req, 0, 500, error); body = JSON.parse(bodystr); body['items'][0]['unknown'] = body['items'][0].delete('create'); { :status => 200, :body => body.to_json, :headers => { 'Content-Type' => 'json' } } })
-  end
-
-  def assert_logs_include(logs, msg, exp_matches=1)
-    matches = logs.grep /#{msg}/
-    assert_equal(exp_matches, matches.length, "Logs do not contain '#{msg}' '#{logs}'")
-  end
-
   def test_configure
     config = %{
       host     logs.google.com
@@ -172,19 +66,34 @@ class ElasticsearchOutput < Test::Unit::TestCase
       user     john
       password doe
     }
-    instance = driver('test', config).instance
+    instance = driver(config).instance
 
     assert_equal 'logs.google.com', instance.host
     assert_equal 777, instance.port
-    assert_equal :https, instance.scheme
+    assert_equal 'https', instance.scheme
     assert_equal '/es/', instance.path
     assert_equal 'john', instance.user
     assert_equal 'doe', instance.password
-    assert_equal :TLSv1, instance.ssl_version
-    assert_nil instance.client_key
-    assert_nil instance.client_cert
-    assert_nil instance.client_key_pass
-    assert_false instance.with_transporter_log
+  end
+
+  test 'lack of tag in chunk_keys' do
+    assert_raise_message(/'tag' in chunk_keys is required./) do
+      driver(Fluent::Config::Element.new(
+               'ROOT', '', {
+                 '@type' => 'elasticsearch',
+                 'host' => 'log.google.com',
+                 'port' => 777,
+                 'scheme' => 'https',
+                 'path' => '/es/',
+                 'user' => 'john',
+                 'pasword' => 'doe',
+               }, [
+                 Fluent::Config::Element.new('buffer', 'mykey', {
+                                               'chunk_keys' => 'mykey'
+                                             }, [])
+               ]
+             ))
+    end
   end
 
   def test_template_already_present
@@ -206,9 +115,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
       to_return(:status => 200, :body => "", :headers => {})
 
-    driver('test', config)
-
-    assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash")
+    driver(config)
   end
 
   def test_template_create
@@ -236,40 +143,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash").
       to_return(:status => 200, :body => "", :headers => {})
 
-    driver('test', config)
-
-    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
-  end
-
-  def test_template_overwrite
-    cwd = File.dirname(__FILE__)
-    template_file = File.join(cwd, 'test_template.json')
-
-    config = %{
-      host            logs.google.com
-      port            777
-      scheme          https
-      path            /es/
-      user            john
-      password        doe
-      template_name   logstash
-      template_file   #{template_file}
-      template_overwrite true
-    }
-
-    # connection start
-    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
-      to_return(:status => 200, :body => "", :headers => {})
-    # check if template exists
-    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash").
-      to_return(:status => 200, :body => "", :headers => {})
-    # creation
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash").
-      to_return(:status => 200, :body => "", :headers => {})
-
-    driver('test', config)
-
-    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
+    driver(config)
   end
 
 
@@ -293,7 +167,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
       to_return(:status => 404, :body => "", :headers => {})
 
     assert_raise(RuntimeError) {
-      driver('test', config)
+      driver(config)
     }
   end
 
@@ -328,51 +202,13 @@ class ElasticsearchOutput < Test::Unit::TestCase
     stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3").
       to_return(:status => 200, :body => "", :headers => {})
 
-    driver('test', config)
+    driver(config)
 
     assert_requested( :put, "https://john:doe@logs.google.com:777/es//_template/logstash1", times: 1)
     assert_requested( :put, "https://john:doe@logs.google.com:777/es//_template/logstash2", times: 1)
     assert_not_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3") #exists
   end
 
-  def test_templates_overwrite
-    cwd = File.dirname(__FILE__)
-    template_file = File.join(cwd, 'test_template.json')
-    config = %{
-      host            logs.google.com
-      port            777
-      scheme          https
-      path            /es/
-      user            john
-      password        doe
-      templates       {"logstash1":"#{template_file}", "logstash2":"#{template_file}","logstash3":"#{template_file}" }
-      template_overwrite true
-    }
-
-    stub_request(:head, "https://john:doe@logs.google.com:777/es//").
-      to_return(:status => 200, :body => "", :headers => {})
-     # check if template exists
-    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash1").
-      to_return(:status => 200, :body => "", :headers => {})
-    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash2").
-      to_return(:status => 200, :body => "", :headers => {})
-    stub_request(:get, "https://john:doe@logs.google.com:777/es//_template/logstash3").
-      to_return(:status => 200, :body => "", :headers => {}) #exists
-
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1").
-      to_return(:status => 200, :body => "", :headers => {})
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
-      to_return(:status => 200, :body => "", :headers => {})
-    stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3").
-      to_return(:status => 200, :body => "", :headers => {})
-
-    driver('test', config)
-
-    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1", times: 1)
-    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2", times: 1)
-    assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash3", times: 1)
-  end
-
   def test_templates_not_used
     cwd = File.dirname(__FILE__)
     template_file = File.join(cwd, 'test_template.json')
@@ -406,7 +242,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     stub_request(:put, "https://john:doe@logs.google.com:777/es//_template/logstash2").
       to_return(:status => 200, :body => "", :headers => {})
 
-    driver('test', config)
+    driver(config)
 
     assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash", times: 1)
 
@@ -440,7 +276,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
       to_return(:status => 200, :body => "", :headers => {})
 
     assert_raise(RuntimeError) {
-      driver('test', config)
+      driver(config)
     }
 
     assert_requested(:put, "https://john:doe@logs.google.com:777/es//_template/logstash1", times: 1)
@@ -454,7 +290,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
       path     /es/
       port     123
     }
-    instance = driver('test', config).instance
+    instance = driver(config).instance
 
     assert_equal 3, instance.get_connection_options[:hosts].length
     host1, host2, host3 = instance.get_connection_options[:hosts]
@@ -476,7 +312,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
       user     default_user
       password default_password
     }
-    instance = driver('test', config).instance
+    instance = driver(config).instance
 
     assert_equal 2, instance.get_connection_options[:hosts].length
     host1, host2 = instance.get_connection_options[:hosts]
@@ -495,39 +331,13 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal '/default_path', host2[:path]
   end
 
-  def test_hosts_list_with_escape_placeholders
-    config = %{
-      hosts    https://%{j+hn}:%{passw@rd}@host1:443/elastic/,http://host2
-      path     /default_path
-      user     default_user
-      password default_password
-    }
-    instance = driver('test', config).instance
-
-    assert_equal 2, instance.get_connection_options[:hosts].length
-    host1, host2 = instance.get_connection_options[:hosts]
-
-    assert_equal 'host1', host1[:host]
-    assert_equal 443, host1[:port]
-    assert_equal 'https', host1[:scheme]
-    assert_equal 'j%2Bhn', host1[:user]
-    assert_equal 'passw%40rd', host1[:password]
-    assert_equal '/elastic/', host1[:path]
-
-    assert_equal 'host2', host2[:host]
-    assert_equal 'http', host2[:scheme]
-    assert_equal 'default_user', host2[:user]
-    assert_equal 'default_password', host2[:password]
-    assert_equal '/default_path', host2[:path]
-  end
-
   def test_single_host_params_and_defaults
     config = %{
       host     logs.google.com
       user     john
       password doe
     }
-    instance = driver('test', config).instance
+    instance = driver(config).instance
 
     assert_equal 1, instance.get_connection_options[:hosts].length
     host1 = instance.get_connection_options[:hosts][0]
@@ -540,63 +350,32 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal nil, host1[:path]
   end
 
-  def test_single_host_params_and_defaults_with_escape_placeholders
-    config = %{
-      host     logs.google.com
-      user     %{j+hn}
-      password %{d@e}
-    }
-    instance = driver('test', config).instance
-
-    assert_equal 1, instance.get_connection_options[:hosts].length
-    host1 = instance.get_connection_options[:hosts][0]
-
-    assert_equal 'logs.google.com', host1[:host]
-    assert_equal 9200, host1[:port]
-    assert_equal 'http', host1[:scheme]
-    assert_equal 'j%2Bhn', host1[:user]
-    assert_equal 'd%40e', host1[:password]
-    assert_equal nil, host1[:path]
-  end
-
   def test_content_type_header
     stub_request(:head, "http://localhost:9200/").
       to_return(:status => 200, :body => "", :headers => {})
-    if Elasticsearch::VERSION >= "6.0.2"
-      elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
-                          with(headers: { "Content-Type" => "application/x-ndjson" })
-    else
-      elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
-                          with(headers: { "Content-Type" => "application/json" })
-    end
-    driver.emit(sample_record)
-    driver.run
+    elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
+      with(headers: { "Content-Type" => "application/json" })
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert_requested(elastic_request)
   end
 
-  def test_write_message_with_bad_chunk
-    driver.configure("target_index_key bad_value\n")
-    log = driver.instance.router.emit_error_handler.log
-    stub_elastic_ping
-    stub_elastic
-    driver.emit({'bad_value'=>"\255"})
-    driver.run
-    assert_logs_include(log.out.logs, /(input string invalid)|(invalid byte sequence in UTF-8)/)
-  end
-
   def test_writes_to_default_index
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert_equal('fluentd', index_cmds.first['index']['_index'])
   end
 
   def test_writes_to_default_type
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert_equal('fluentd', index_cmds.first['index']['_type'])
   end
 
@@ -604,8 +383,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("index_name myindex\n")
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert_equal('myindex', index_cmds.first['index']['_index'])
   end
 
@@ -613,8 +393,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("index_name MyIndex\n")
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     # Allthough index_name has upper-case characters,
     # it should be set as lower-case when sent to elasticsearch.
     assert_equal('myindex', index_cmds.first['index']['_index'])
@@ -625,8 +406,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
     stub_elastic_ping
     stub_elastic
     record = sample_record.clone
-    driver.emit(sample_record.merge('@target_index' => 'local-override'))
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record.merge('@target_index' => 'local-override'))
+    end
     assert_equal('local-override', index_cmds.first['index']['_index'])
     assert_nil(index_cmds[1]['@target_index'])
   end
@@ -634,55 +416,49 @@ class ElasticsearchOutput < Test::Unit::TestCase
   def test_writes_to_target_index_key_logstash
     driver.configure("target_index_key @target_index
                       logstash_format true")
-    time = Time.parse Date.today.iso8601
+    time = Time.parse Date.today.to_s
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record.merge('@target_index' => 'local-override'), time.to_i)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(time.to_i, sample_record.merge('@target_index' => 'local-override'))
+    end
     assert_equal('local-override', index_cmds.first['index']['_index'])
   end
 
    def test_writes_to_target_index_key_logstash_uppercase
     driver.configure("target_index_key @target_index
                       logstash_format true")
-    time = Time.parse Date.today.iso8601
+    time = Time.parse Date.today.to_s
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record.merge('@target_index' => 'Local-Override'), time.to_i)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(time.to_i, sample_record.merge('@target_index' => 'local-override'))
+    end
     # Allthough @target_index has upper-case characters,
     # it should be set as lower-case when sent to elasticsearch.
     assert_equal('local-override', index_cmds.first['index']['_index'])
   end
 
-  def test_writes_to_default_index_with_pipeline
-    pipeline = "fluentd"
-    driver.configure("pipeline #{pipeline}")
-    stub_elastic_ping
-    stub_elastic
-    driver.emit(sample_record)
-    driver.run
-    assert_equal(pipeline, index_cmds.first['index']['pipeline'])
-  end
-
   def test_writes_to_target_index_key_fallack
     driver.configure("target_index_key @target_index\n")
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert_equal('fluentd', index_cmds.first['index']['_index'])
   end
 
   def test_writes_to_target_index_key_fallack_logstash
     driver.configure("target_index_key @target_index\n
                       logstash_format true")
-    time = Time.parse Date.today.iso8601
+    time = Time.parse Date.today.to_s
     logstash_index = "logstash-#{time.getutc.strftime("%Y.%m.%d")}"
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record, time.to_i)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(time.to_i, sample_record)
+    end
     assert_equal(logstash_index, index_cmds.first['index']['_index'])
   end
 
@@ -690,8 +466,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("type_name mytype\n")
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert_equal('mytype', index_cmds.first['index']['_type'])
   end
 
@@ -700,8 +477,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
     stub_elastic_ping
     stub_elastic
     record = sample_record.clone
-    driver.emit(sample_record.merge('@target_type' => 'local-override'))
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record.merge('@target_type' => 'local-override'))
+    end
     assert_equal('local-override', index_cmds.first['index']['_type'])
     assert_nil(index_cmds[1]['@target_type'])
   end
@@ -710,8 +488,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("target_type_key @target_type\n")
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert_equal('fluentd', index_cmds.first['index']['_type'])
   end
 
@@ -720,8 +499,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
                       type_name mytype")
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert_equal('mytype', index_cmds.first['index']['_type'])
   end
 
@@ -729,12 +509,13 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("target_type_key kubernetes.labels.log_type\n")
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record.merge('kubernetes' => {
-      'labels' => {
-        'log_type' => 'local-override'
-      }
-    }))
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record.merge('kubernetes' => {
+        'labels' => {
+          'log_type' => 'local-override'
+        }
+      }))
+    end
     assert_equal('local-override', index_cmds.first['index']['_type'])
     assert_nil(index_cmds[1]['kubernetes']['labels']['log_type'])
   end
@@ -743,12 +524,13 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("target_type_key kubernetes.labels.log_type\n")
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record.merge('kubernetes' => {
-      'labels' => {
-        'other_labels' => 'test'
-      }
-    }))
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record.merge('kubernetes' => {
+        'labels' => {
+          'other_labels' => 'test'
+        }
+      }))
+    end
     assert_equal('fluentd', index_cmds.first['index']['_type'])
   end
 
@@ -756,8 +538,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("host 192.168.33.50\n")
     stub_elastic_ping("http://192.168.33.50:9200")
     elastic_request = stub_elastic("http://192.168.33.50:9200/_bulk")
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert_requested(elastic_request)
   end
 
@@ -765,8 +548,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("port 9201\n")
     stub_elastic_ping("http://localhost:9201")
     elastic_request = stub_elastic("http://localhost:9201/_bulk")
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert_requested(elastic_request)
   end
 
@@ -782,12 +566,12 @@ class ElasticsearchOutput < Test::Unit::TestCase
       stub_elastic_with_store_index_command_counts("http://#{host}:#{port}/_bulk")
     end
 
-    1000.times do
-      driver.emit(sample_record.merge('age'=>rand(100)))
+    driver.run(default_tag: 'test') do
+      1000.times do
+        driver.feed(sample_record.merge('age'=>rand(100)))
+      end
     end
 
-    driver.run
-
     # @note: we cannot make multi chunks with options (flush_interval, buffer_chunk_limit)
     # it's Fluentd test driver's constraint
     # so @index_command_counts.size is always 1
@@ -817,8 +601,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
     stub_elastic_ping
     stub_elastic
-    driver.emit(original_hash)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(original_hash)
+    end
     assert_equal expected_output, index_cmds[1]
   end
 
@@ -830,26 +615,29 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
     stub_elastic_ping
     stub_elastic
-    driver.emit(original_hash)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(original_hash)
+    end
     assert_equal expected_output, index_cmds[1]
   end
 
   def test_makes_bulk_request
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.emit(sample_record.merge('age' => 27))
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+      driver.feed(sample_record.merge('age' => 27))
+    end
     assert_equal(4, index_cmds.count)
   end
 
   def test_all_records_are_preserved_in_bulk
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.emit(sample_record.merge('age' => 27))
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+      driver.feed(sample_record.merge('age' => 27))
+    end
     assert_equal(26, index_cmds[1]['age'])
     assert_equal(27, index_cmds[3]['age'])
   end
@@ -862,8 +650,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
     logstash_index = "logstash-2015.05.31"
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record, dt.to_time.to_i)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(dt.to_time.to_i, sample_record)
+    end
     assert_equal(logstash_index, index_cmds.first['index']['_index'])
   end
 
@@ -876,46 +665,35 @@ class ElasticsearchOutput < Test::Unit::TestCase
     index = "logstash-#{time.strftime("%Y.%m.%d")}"
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record, time.to_i)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(time.to_i, sample_record)
+    end
     assert_equal(index, index_cmds.first['index']['_index'])
   end
 
   def test_writes_to_logstash_index_with_specified_prefix
     driver.configure("logstash_format true
                       logstash_prefix myprefix")
-    time = Time.parse Date.today.iso8601
+    time = Time.parse Date.today.to_s
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record, time.to_i)
-    driver.run
-    assert_equal(logstash_index, index_cmds.first['index']['_index'])
-  end
-
-  def test_writes_to_logstash_index_with_specified_prefix_and_separator
-    separator = '_'
-    driver.configure("logstash_format true
-                      logstash_prefix_separator #{separator}
-                      logstash_prefix myprefix")
-    time = Time.parse Date.today.iso8601
-    logstash_index = "myprefix#{separator}#{time.getutc.strftime("%Y.%m.%d")}"
-    stub_elastic_ping
-    stub_elastic
-    driver.emit(sample_record, time.to_i)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(time.to_i, sample_record)
+    end
     assert_equal(logstash_index, index_cmds.first['index']['_index'])
   end
 
   def test_writes_to_logstash_index_with_specified_prefix_uppercase
     driver.configure("logstash_format true
                       logstash_prefix MyPrefix")
-    time = Time.parse Date.today.iso8601
+    time = Time.parse Date.today.to_s
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record, time.to_i)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(time.to_i, sample_record)
+    end
     # Allthough logstash_prefix has upper-case characters,
     # it should be set as lower-case when sent to elasticsearch.
     assert_equal(logstash_index, index_cmds.first['index']['_index'])
@@ -924,12 +702,13 @@ class ElasticsearchOutput < Test::Unit::TestCase
     def test_writes_to_logstash_index_with_specified_dateformat
     driver.configure("logstash_format true
                       logstash_dateformat %Y.%m")
-    time = Time.parse Date.today.iso8601
+    time = Time.parse Date.today.to_s
     logstash_index = "logstash-#{time.getutc.strftime("%Y.%m")}"
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record, time.to_i)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(time.to_i, sample_record)
+    end
     assert_equal(logstash_index, index_cmds.first['index']['_index'])
   end
 
@@ -937,64 +716,66 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("logstash_format true
                       logstash_prefix myprefix
                       logstash_dateformat %Y.%m")
-    time = Time.parse Date.today.iso8601
+    time = Time.parse Date.today.to_s
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m")}"
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record, time.to_i)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(time.to_i, sample_record)
+    end
     assert_equal(logstash_index, index_cmds.first['index']['_index'])
   end
 
+  def test_error_if_tag_not_in_chunk_keys
+    assert_raise(Fluent::ConfigError) {
+      config = %{
+        <buffer foo>
+        </buffer>
+      }
+      driver.configure(config)
+    }
+  end
+
+  def test_can_use_custom_chunk_along_with_tag
+    config = %{
+      <buffer tag, foo>
+      </buffer>
+    }
+    driver.configure(config)
+  end
+
   def test_doesnt_add_logstash_timestamp_by_default
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert_nil(index_cmds[1]['@timestamp'])
   end
 
-  def test_adds_timestamp_when_logstash
+  def test_adds_logstash_timestamp_when_configured
     driver.configure("logstash_format true\n")
     stub_elastic_ping
     stub_elastic
-    ts = DateTime.now.iso8601
-    driver.emit(sample_record)
-    driver.run
-    assert(index_cmds[1].has_key? '@timestamp')
-    assert_equal(index_cmds[1]['@timestamp'], ts)
-  end
-
-  def test_adds_timestamp_when_include_timestamp
-    driver.configure("include_timestamp true\n")
-    stub_elastic_ping
-    stub_elastic
     ts = DateTime.now
-    time = ts.to_time
-    driver.emit(sample_record, time)
-    driver.run
+    time = Fluent::EventTime.from_time(ts.to_time)
+    driver.run(default_tag: 'test') do
+      driver.feed(time, sample_record)
+    end
+    tf = "%Y-%m-%dT%H:%M:%S%:z"
+    timef = Fluent::TimeFormatter.new(tf, true, ENV["TZ"])
     assert(index_cmds[1].has_key? '@timestamp')
-    assert_equal(index_cmds[1]['@timestamp'], ts.iso8601)
+    assert_equal(timef.format(Time.parse(index_cmds[1]['@timestamp'])).to_s, ts.to_s)
   end
 
   def test_uses_custom_timestamp_when_included_in_record
     driver.configure("logstash_format true\n")
     stub_elastic_ping
     stub_elastic
-    ts = DateTime.new(2001,2,3).iso8601
-    driver.emit(sample_record.merge!('@timestamp' => ts))
-    driver.run
-    assert(index_cmds[1].has_key? '@timestamp')
-    assert_equal(index_cmds[1]['@timestamp'], ts)
-  end
-
-  def test_uses_custom_timestamp_when_included_in_record_without_logstash
-    driver.configure("include_timestamp true\n")
-    stub_elastic_ping
-    stub_elastic
-    ts = DateTime.new(2001,2,3).iso8601
-    driver.emit(sample_record.merge!('@timestamp' => ts))
-    driver.run
+    ts = DateTime.new(2001,2,3).to_s
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record.merge!('@timestamp' => ts))
+    end
     assert(index_cmds[1].has_key? '@timestamp')
     assert_equal(index_cmds[1]['@timestamp'], ts)
   end
@@ -1004,40 +785,12 @@ class ElasticsearchOutput < Test::Unit::TestCase
                       time_key vtm\n")
     stub_elastic_ping
     stub_elastic
-    ts = DateTime.new(2001,2,3).iso8601
-    driver.emit(sample_record.merge!('vtm' => ts))
-    driver.run
-    assert(index_cmds[1].has_key? '@timestamp')
-    assert_equal(index_cmds[1]['@timestamp'], ts)
-  end
-
-  def test_uses_custom_time_key_with_format
-    driver.configure("logstash_format true
-                      time_key_format %Y-%m-%d %H:%M:%S.%N%z
-                      time_key vtm\n")
-    stub_elastic_ping
-    stub_elastic
-    ts = "2001-02-03 13:14:01.673+02:00"
-    driver.emit(sample_record.merge!('vtm' => ts))
-    driver.run
-    assert(index_cmds[1].has_key? '@timestamp')
-    assert_equal(index_cmds[1]['@timestamp'], ts)
-    assert_equal("logstash-2001.02.03", index_cmds[0]['index']['_index'])
-  end
-
-  def test_uses_custom_time_key_with_format_without_logstash
-    driver.configure("include_timestamp true
-                      index_name test
-                      time_key_format %Y-%m-%d %H:%M:%S.%N%z
-                      time_key vtm\n")
-    stub_elastic_ping
-    stub_elastic
-    ts = "2001-02-03 13:14:01.673+02:00"
-    driver.emit(sample_record.merge!('vtm' => ts))
-    driver.run
+    ts = DateTime.new(2001,2,3).to_s
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record.merge!('vtm' => ts))
+    end
     assert(index_cmds[1].has_key? '@timestamp')
     assert_equal(index_cmds[1]['@timestamp'], ts)
-    assert_equal("test", index_cmds[0]['index']['_index'])
   end
 
   def test_uses_custom_time_key_exclude_timekey
@@ -1046,9 +799,10 @@ class ElasticsearchOutput < Test::Unit::TestCase
                       time_key_exclude_timestamp true\n")
     stub_elastic_ping
     stub_elastic
-    ts = DateTime.new(2001,2,3).iso8601
-    driver.emit(sample_record.merge!('vtm' => ts))
-    driver.run
+    ts = DateTime.new(2001,2,3).to_s
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record.merge!('vtm' => ts))
+    end
     assert(!index_cmds[1].key?('@timestamp'), '@timestamp should be messing')
   end
 
@@ -1058,27 +812,14 @@ class ElasticsearchOutput < Test::Unit::TestCase
     stub_elastic_ping
     stub_elastic
     ts = "2001-02-03T13:14:01.673+02:00"
-    driver.emit(sample_record.merge!('@timestamp' => ts))
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record.merge!('@timestamp' => ts))
+    end
     assert_equal("logstash-2001.02.03", index_cmds[0]['index']['_index'])
     assert(index_cmds[1].has_key? '@timestamp')
     assert_equal(index_cmds[1]['@timestamp'], ts)
   end
 
-  def test_uses_custom_time_key_format_without_logstash
-    driver.configure("include_timestamp true
-                      index_name test
-                      time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n")
-    stub_elastic_ping
-    stub_elastic
-    ts = "2001-02-03T13:14:01.673+02:00"
-    driver.emit(sample_record.merge!('@timestamp' => ts))
-    driver.run
-    assert_equal("test", index_cmds[0]['index']['_index'])
-    assert(index_cmds[1].has_key? '@timestamp')
-    assert_equal(index_cmds[1]['@timestamp'], ts)
-  end
-
   data(:default => nil,
        :custom_tag => 'es_plugin.output.time.error')
   def test_uses_custom_time_key_format_logs_an_error(tag_for_error)
@@ -1092,12 +833,11 @@ class ElasticsearchOutput < Test::Unit::TestCase
     ts = "2001/02/03 13:14:01,673+02:00"
     index = "logstash-#{Date.today.strftime("%Y.%m.%d")}"
 
-    driver.emit(sample_record.merge!('@timestamp' => ts))
-    driver.run
-
-    log = driver.instance.router.emit_error_handler.log
-    errors = log.out.logs.grep /tag="#{tag_for_error}"/
-    assert_equal(1, errors.length, "Error was logged for timestamp parse failure")
+    flexmock(driver.instance.router).should_receive(:emit_error_event)
+      .with(tag_for_error, Fluent::EventTime, Hash, ArgumentError).once
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record.merge!('@timestamp' => ts))
+    end
 
     assert_equal(index, index_cmds[0]['index']['_index'])
     assert(index_cmds[1].has_key? '@timestamp')
@@ -1111,14 +851,15 @@ class ElasticsearchOutput < Test::Unit::TestCase
     stub_elastic_ping
     stub_elastic
     ts = "Thu Nov 29 14:33:20 GMT 2001"
-    driver.emit(sample_record.merge!('@timestamp' => ts))
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record.merge!('@timestamp' => ts))
+    end
     assert_equal("logstash-2001.11.29", index_cmds[0]['index']['_index'])
     assert(index_cmds[1].has_key? '@timestamp')
     assert_equal(index_cmds[1]['@timestamp'], ts)
   end
 
-  def test_uses_no_subsecond_precision_by_default
+  def test_uses_nanosecond_precision_by_default
     driver.configure("logstash_format true\n")
     stub_elastic_ping
     stub_elastic
@@ -1127,10 +868,11 @@ class ElasticsearchOutput < Test::Unit::TestCase
     rescue
       time = Fluent::Engine.now
     end
-    driver.emit(sample_record, time)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(time.to_i, sample_record)
+    end
     assert(index_cmds[1].has_key? '@timestamp')
-    assert_equal(index_cmds[1]['@timestamp'], Time.at(time).iso8601)
+    assert_equal(index_cmds[1]['@timestamp'], Time.at(time).iso8601(9))
   end
 
   def test_uses_subsecond_precision_when_configured
@@ -1143,8 +885,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
     rescue
       time = Fluent::Engine.now
     end
-    driver.emit(sample_record, time)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(time.to_i, sample_record)
+    end
     assert(index_cmds[1].has_key? '@timestamp')
     assert_equal(index_cmds[1]['@timestamp'], Time.at(time).iso8601(3))
   end
@@ -1152,17 +895,19 @@ class ElasticsearchOutput < Test::Unit::TestCase
   def test_doesnt_add_tag_key_by_default
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert_nil(index_cmds[1]['tag'])
   end
 
   def test_adds_tag_key_when_configured
-    driver('mytag').configure("include_tag_key true\n")
+    driver.configure("include_tag_key true\n")
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'mytag') do
+      driver.feed(sample_record)
+    end
     assert(index_cmds[1].has_key?('tag'))
     assert_equal(index_cmds[1]['tag'], 'mytag')
   end
@@ -1171,8 +916,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("id_key request_id\n")
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert_equal(index_cmds[0]['index']['_id'], '42')
   end
 
@@ -1180,16 +926,18 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("id_key another_request_id\n")
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert(!index_cmds[0]['index'].has_key?('_id'))
   end
 
   def test_adds_id_key_when_not_configured
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert(!index_cmds[0]['index'].has_key?('_id'))
   end
 
@@ -1197,8 +945,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("parent_key parent_id\n")
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert_equal(index_cmds[0]['index']['_parent'], 'parent')
   end
 
@@ -1206,16 +955,18 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("parent_key another_parent_id\n")
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert(!index_cmds[0]['index'].has_key?('_parent'))
   end
 
   def test_adds_parent_key_when_not_configured
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert(!index_cmds[0]['index'].has_key?('_parent'))
   end
 
@@ -1223,8 +974,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("routing_key routing_id\n")
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert_equal(index_cmds[0]['index']['_routing'], 'routing')
   end
 
@@ -1232,16 +984,18 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("routing_key another_routing_id\n")
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert(!index_cmds[0]['index'].has_key?('_routing'))
   end
 
   def test_adds_routing_key_when_not_configured
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert(!index_cmds[0]['index'].has_key?('_routing'))
   end
 
@@ -1249,8 +1003,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("remove_keys key1\n")
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record.merge('key1' => 'v1', 'key2' => 'v2'))
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record.merge('key1' => 'v1', 'key2' => 'v2'))
+    end
     assert(!index_cmds[1].has_key?('key1'))
     assert(index_cmds[1].has_key?('key2'))
   end
@@ -1259,8 +1014,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("remove_keys key1, key2\n")
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record.merge('key1' => 'v1', 'key2' => 'v2'))
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record.merge('key1' => 'v1', 'key2' => 'v2'))
+    end
     assert(!index_cmds[1].has_key?('key1'))
     assert(!index_cmds[1].has_key?('key2'))
   end
@@ -1268,19 +1024,13 @@ class ElasticsearchOutput < Test::Unit::TestCase
   def test_request_error
     stub_elastic_ping
     stub_elastic_unavailable
-    driver.emit(sample_record)
     assert_raise(Elasticsearch::Transport::Transport::Errors::ServiceUnavailable) {
-      driver.run
+      driver.run(default_tag: 'test') do
+        driver.feed(sample_record)
+      end
     }
   end
 
-  def test_garbage_record_error
-    stub_elastic_ping
-    stub_elastic
-    driver.emit("some garbage string")
-    driver.run
-  end
-
   def test_connection_failed_retry
     connection_resets = 0
 
@@ -1292,11 +1042,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
       raise Faraday::ConnectionFailed, "Test message"
     end
 
-    driver.emit(sample_record)
-
-    assert_raise(Fluent::ElasticsearchOutput::ConnectionFailure) {
-      driver.run
-    }
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert_equal(connection_resets, 3)
   end
 
@@ -1312,16 +1060,21 @@ class ElasticsearchOutput < Test::Unit::TestCase
     end
 
     driver.configure("reconnect_on_error true\n")
-    driver.emit(sample_record)
 
     assert_raise(ZeroDivisionError) {
-      driver.run
+      driver.run(default_tag: 'test', shutdown: false) do
+        driver.feed(sample_record)
+      end
     }
 
-    assert_raise(ZeroDivisionError) {
-      driver.run
+    assert_raise(Timeout::Error) {
+      driver.run(default_tag: 'test', shutdown: false) do
+        driver.feed(sample_record)
+      end
     }
-    assert_equal(connection_resets, 2)
+    # FIXME: Consider keywords arguments in #run and how to test this later.
+    # Because v0.14 test driver does not have 1 to 1 correspondence between #run and #flush in tests.
+    assert_equal(connection_resets, 1)
   end
 
   def test_reconnect_on_error_disabled
@@ -1336,219 +1089,28 @@ class ElasticsearchOutput < Test::Unit::TestCase
     end
 
     driver.configure("reconnect_on_error false\n")
-    driver.emit(sample_record)
 
     assert_raise(ZeroDivisionError) {
-      driver.run
+      driver.run(default_tag: 'test', shutdown: false) do
+        driver.feed(sample_record)
+      end
     }
 
-    assert_raise(ZeroDivisionError) {
-      driver.run
+    assert_raise(Timeout::Error) {
+      driver.run(default_tag: 'test', shutdown: false) do
+        driver.feed(sample_record)
+      end
     }
     assert_equal(connection_resets, 1)
   end
 
-  def test_bulk_error_retags_when_configured
-    driver.configure("retry_tag retry\n")
-    stub_elastic_ping
-    stub_request(:post, 'http://localhost:9200/_bulk')
-        .to_return(lambda do |req|
-      { :status => 200,
-        :headers => { 'Content-Type' => 'json' },
-        :body => %({
-          "took" : 1,
-          "errors" : true,
-          "items" : [
-            {
-              "create" : {
-                "_index" : "foo",
-                "_type"  : "bar",
-                "_id" : "abc",
-                "status" : 500,
-                "error" : {
-                  "type" : "some unrecognized type",
-                  "reason":"some error to cause version mismatch"
-                }
-              }
-            }
-           ]
-        })
-     }
-    end)
-    driver.emit(sample_record, 1)
-
-    driver.expect_emit('retry', 1, sample_record)
-    driver.run
-  end
-
-  def test_create_should_write_records_with_ids_and_skip_those_without
-    driver.configure("write_operation create\nid_key my_id\nlog_level debug")
-    log = driver.instance.router.emit_error_handler.log
-    stub_elastic_ping
-    stub_request(:post, 'http://localhost:9200/_bulk')
-        .to_return(lambda do |req|
-      { :status => 200,
-        :headers => { 'Content-Type' => 'json' },
-        :body => %({
-          "took" : 1,
-          "errors" : true,
-          "items" : [
-            {
-              "create" : {
-                "_index" : "foo",
-                "_type"  : "bar",
-                "_id" : "abc"
-              }
-            },
-            {
-              "create" : {
-                "_index" : "foo",
-                "_type"  : "bar",
-                "_id" : "xyz",
-                "status" : 500,
-                "error" : {
-                  "type" : "some unrecognized type",
-                  "reason":"some error to cause version mismatch"
-                }
-              }
-            }
-           ]
-        })
-     }
-    end)
-    sample_record1 = sample_record('my_id' => 'abc')
-    sample_record4 = sample_record('my_id' => 'xyz')
-    driver.emit(sample_record1, 1)
-    driver.emit(sample_record, 2)
-    driver.emit(sample_record, 3)
-    driver.emit(sample_record4, 4)
-
-    # one record succeeded while the other should be 'retried'
-    driver.expect_emit('test', 4, sample_record4)
-    driver.run
-    assert_logs_include(log.out.logs, /(Dropping record)/, 2)
-  end
-
-  def test_create_should_write_records_with_ids_and_emit_those_without
-    driver.configure("write_operation create\nid_key my_id\nemit_error_for_missing_id true")
-    log = driver.instance.router.emit_error_handler.log
-    stub_elastic_ping
-    stub_request(:post, 'http://localhost:9200/_bulk')
-        .to_return(lambda do |req|
-      { :status => 200,
-        :headers => { 'Content-Type' => 'json' },
-        :body => %({
-          "took" : 1,
-          "errors" : true,
-          "items" : [
-            {
-              "create" : {
-                "_index" : "foo",
-                "_type"  : "bar",
-                "_id" : "abc"
-              }
-            },
-            {
-              "create" : {
-                "_index" : "foo",
-                "_type"  : "bar",
-                "_id" : "xyz",
-                "status" : 500,
-                "error" : {
-                  "type" : "some unrecognized type",
-                  "reason":"some error to cause version mismatch"
-                }
-              }
-            }
-           ]
-        })
-     }
-    end)
-    sample_record1 = sample_record('my_id' => 'abc')
-    sample_record4 = sample_record('my_id' => 'xyz')
-    driver.emit(sample_record1, 1)
-    driver.emit(sample_record, 2)
-    driver.emit(sample_record, 3)
-    driver.emit(sample_record4, 4)
-
-    # one record succeeded while the other should be 'retried'
-    driver.expect_emit('test', 4, sample_record4)
-    driver.run
-    assert_logs_include(log.out.logs, /(Missing '_id' field)/, 2)
-  end
-
-  def test_bulk_error
-    stub_elastic_ping
-    stub_request(:post, 'http://localhost:9200/_bulk')
-        .to_return(lambda do |req|
-      { :status => 200,
-        :headers => { 'Content-Type' => 'json' },
-        :body => %({
-          "took" : 1,
-          "errors" : true,
-          "items" : [
-            {
-              "create" : {
-                "_index" : "foo",
-                "_type"  : "bar",
-                "_id" : "abc",
-                "status" : 500,
-                "error" : {
-                  "type" : "some unrecognized type",
-                  "reason":"some error to cause version mismatch"
-                }
-              }
-            },
-            {
-              "create" : {
-                "_index" : "foo",
-                "_type"  : "bar",
-                "_id" : "abc",
-                "status" : 201
-              }
-            },
-            {
-              "create" : {
-                "_index" : "foo",
-                "_type"  : "bar",
-                "_id" : "abc",
-                "status" : 500,
-                "error" : {
-                  "type" : "some unrecognized type",
-                  "reason":"some error to cause version mismatch"
-                }
-              }
-            },
-            {
-              "create" : {
-                "_index" : "foo",
-                "_type"  : "bar",
-                "_id" : "abc",
-                "_id" : "abc",
-                "status" : 409
-              }
-            }
-           ]
-        })
-     }
-    end)
-
-    driver.emit(sample_record, 1)
-    driver.emit(sample_record, 2)
-    driver.emit(sample_record, 3)
-    driver.emit(sample_record, 4)
-
-    driver.expect_emit('test', 1, sample_record)
-    driver.expect_emit('test', 3, sample_record)
-    driver.run
-  end
-
   def test_update_should_not_write_if_theres_no_id
     driver.configure("write_operation update\n")
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert_nil(index_cmds)
   end
 
@@ -1556,8 +1118,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("write_operation upsert\n")
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert_nil(index_cmds)
   end
 
@@ -1565,8 +1128,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("write_operation create\n")
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert_nil(index_cmds)
   end
 
@@ -1575,8 +1139,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
                       id_key request_id")
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert(index_cmds[0].has_key?("update"))
     assert(!index_cmds[1]["doc_as_upsert"])
     assert(!index_cmds[1]["upsert"])
@@ -1588,8 +1153,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
                       remove_keys_on_update parent_id")
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert(index_cmds[1]["doc"])
     assert(!index_cmds[1]["doc"]["parent_id"])
   end
@@ -1599,8 +1165,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
                       id_key request_id")
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert(index_cmds[0].has_key?("update"))
     assert(index_cmds[1]["doc_as_upsert"])
     assert(!index_cmds[1]["upsert"])
@@ -1612,8 +1179,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
                       remove_keys_on_update parent_id")
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert(index_cmds[0].has_key?("update"))
     assert(!index_cmds[1]["doc_as_upsert"])
     assert(index_cmds[1]["upsert"])
@@ -1626,8 +1194,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
                       remove_keys_on_update parent_id")
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert(index_cmds[1]["upsert"] != index_cmds[1]["doc"])
     assert(!index_cmds[1]["doc"]["parent_id"])
     assert(index_cmds[1]["upsert"]["parent_id"])
@@ -1639,8 +1208,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
                       remove_keys_on_update foo,baz")
     stub_elastic_ping
     stub_elastic
-    driver.emit("id" => 1, "foo" => "bar", "baz" => "quix", "zip" => "zam")
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed("id" => 1, "foo" => "bar", "baz" => "quix", "zip" => "zam")
+    end
     assert(
       index_cmds[1]["doc"] == {
         "id" => 1,
@@ -1663,8 +1233,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
                       remove_keys_on_update_key keys_to_skip")
     stub_elastic_ping
     stub_elastic
-    driver.emit("id" => 1, "foo" => "bar", "baz" => "quix", "keys_to_skip" => ["baz"])
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed("id" => 1, "foo" => "bar", "baz" => "quix", "keys_to_skip" => ["baz"])
+    end
     assert(
       index_cmds[1]["doc"] == {
         "id" => 1,
@@ -1687,8 +1258,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
                       remove_keys_on_update_key keys_to_skip")
     stub_elastic_ping
     stub_elastic
-    driver.emit("id" => 1, "foo" => "bar", "baz" => "quix", "keys_to_skip" => ["baz"])
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed("id" => 1, "foo" => "bar", "baz" => "quix", "keys_to_skip" => ["baz"])
+    end
     assert(
       index_cmds[1]["doc"] == {
         "id" => 1,
@@ -1711,84 +1283,9 @@ class ElasticsearchOutput < Test::Unit::TestCase
                       id_key request_id")
     stub_elastic_ping
     stub_elastic
-    driver.emit(sample_record)
-    driver.run
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
     assert(index_cmds[0].has_key?("create"))
   end
-
-  def test_use_simple_sniffer
-    require 'fluent/plugin/elasticsearch_simple_sniffer'
-    driver.configure("sniffer_class_name Fluent::ElasticsearchSimpleSniffer
-                      log_level debug
-                      with_transporter_log true
-                      reload_connections true
-                      reload_after 1")
-    stub_elastic_ping
-    stub_elastic
-    driver.emit(sample_record)
-    driver.run
-    log = driver.instance.router.emit_error_handler.log
-    # 2 - one for the ping, one for the _bulk
-    assert_logs_include(log.out.logs, /In Fluent::ElasticsearchSimpleSniffer hosts/, 2)
-  end
-
-  def test_suppress_doc_wrap
-    driver.configure('write_operation update
-                      id_key id
-                      remove_keys id
-                      suppress_doc_wrap true')
-    stub_elastic_ping
-    stub_elastic
-    doc_body = {'field' => 'value'}
-    doc_record = {'id' => 1, 'doc' => doc_body}
-    script_body = {'source' => 'ctx._source.counter += params.param1',
-                   'lang' => 'painless',
-                   'params' => {'param1' => 1}}
-    upsert_body = {'counter' => 1}
-    script_record = {'id' => 2, 'script' => script_body, 'upsert' => upsert_body}
-    driver.emit(doc_record, 1)
-    driver.emit(script_record, 2)
-    driver.run
-    assert(
-      index_cmds[1] == {'doc' => doc_body}
-    )
-    assert(
-      index_cmds[3] == {
-        'script' => script_body,
-        'upsert' => upsert_body
-      }
-    )
-  end
-
-  def test_suppress_doc_wrap_should_handle_record_as_is_at_upsert
-    driver.configure('write_operation upsert
-                      id_key id
-                      remove_keys id
-                      suppress_doc_wrap true')
-    stub_elastic_ping
-    stub_elastic
-    doc_body = {'field' => 'value'}
-    doc_record = {'id' => 1, 'doc' => doc_body, 'doc_as_upsert' => true}
-    script_body = {'source' => 'ctx._source.counter += params.param1',
-                   'lang' => 'painless',
-                   'params' => {'param1' => 1}}
-    upsert_body = {'counter' => 1}
-    script_record = {'id' => 2, 'script' => script_body, 'upsert' => upsert_body}
-    driver.emit(doc_record, 1)
-    driver.emit(script_record, 2)
-    driver.run
-    assert(
-      index_cmds[1] == {
-        'doc' => doc_body,
-        'doc_as_upsert' => true
-      }
-    )
-    assert(
-      index_cmds[3] == {
-        'script' => script_body,
-        'upsert' => upsert_body
-      }
-    )
-  end
-
 end