fluent-plugin-elasticsearch 2.12.5 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2e61b71d57d8dcc1f75180c40f8e926804b694004b30cb0ecff73407fb1c647c
-  data.tar.gz: b1503054bd7fe71e15bc3aa0c9f0fd90e320926ae40138924ddae9511e1bde8a
+  metadata.gz: c5f8aa97feb4191caf5f4bb14fcdd6de7e453ccabee8b0fc50a3b256e56ebc93
+  data.tar.gz: 35df640f4f7c7944ea2190fe4627ad2385c102c551fcf3b8bbf25984a00bdd58
 SHA512:
-  metadata.gz: ccf3115fb1d7e434cb7bf986bc4c3f2ef91af0eeeddab5b49bf5603089b4ce48919eb56a60a5f07222be8695caf03ea1d9017ff54a6e0c8e96fcd7adec2dfbec
-  data.tar.gz: a5a2b1ee4a2bcf33406e3c754547d0b8de41082a334c196c4ab7b4a9f410770bcf018064ddf5375ceea9e5e935a23e5d88b3b9e17e4a27e9cfc43e1fa6d1acee
+  metadata.gz: 9bb28681d6b85d4260a4797a244c37cfcedd4f23a1d0b5b690df45845c287e0c20741033eb4f6289d7ed2bff5517d0749b5258361fe056bdf628445d69979920
+  data.tar.gz: 2947d2e1fa50a87b507b1e8792a254ae0d595472e40553d0d14ec74a533c2be8c1fcbc59347c098440354f5949e3e07bbf4f7aa397ed3fd2c3a7ff16e692aacd

data/History.md

@@ -2,6 +2,11 @@
 
 ### [Unreleased]
 
+### 3.0.0
+- Use fluentd core retry mechanism (#519)
+- Depends on builtin retrying mechanism (#518)
+- Loosen ConnectionRetryFailure condition when flush_thread_count > 1 and depends on Fluentd core retrying mechanism (#516)
+
 ### 2.12.5
 - Ensure sniffer class constants definition before calling #client (#515)
 

data/README.md

@@ -83,6 +83,7 @@ Current maintainers: @cosmo0920
   + [Cannot see detailed failure log](#cannot-see-detailed-failure-log)
   + [Cannot connect TLS enabled reverse Proxy](#cannot-connect-tls-enabled-reverse-proxy)
   + [Declined logs are resubmitted forever, why?](#declined-logs-are-resubmitted-forever-why)
+  + [Suggested to increase flush_thread_count, why?](#suggested-to-increase-flush_thread_count-why)
 * [Contact](#contact)
 * [Contributing](#contributing)
 * [Running tests](#running-tests)
@@ -1226,6 +1227,30 @@ The following configuration uses label:
 </label>
 ```
 
+### Suggested to increase flush_thread_count, why?
+
+fluent-plugin-elasticsearch default behavior has a possibility to cause events traffic jam.
+When users use `flush_thread_count` = 1, ES plugin retries to send events if connection errors are disappeared.
+
+To prevent the following warning and sending events blocking, you must specify `flush_thread_count` > 2:
+
+```log
+2018-12-24 14:32:06 +0900 [warn]: #0 To prevent events traffic jam, you should specify 2 or more 'flush_thread_count'.
+```
+
+```aconf
+<match out.elasticsearch.**>
+  @type elasticsearch
+  host localhost
+  port 9200
+  # ...
+  <buffer tag>
+    @type memory # or file
+    flush_thread_count 4
+  </buffer>
+</match>
+```
+
 ## Contact
 
 If you have a question, [open an Issue](https://github.com/uken/fluent-plugin-elasticsearch/issues).

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '2.12.5'
+  s.version       = '3.0.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -1,4 +1,7 @@
+require 'fluent/error'
+
 module Fluent::ElasticsearchIndexTemplate
+  class TemplateInstallationFailure < Fluent::UnrecoverableError; end
 
   def get_template(template_file)
     if !File.exists?(template_file)
@@ -31,7 +34,7 @@ module Fluent::ElasticsearchIndexTemplate
     retries = 0
     begin
       yield
-    rescue Fluent::Plugin::ElasticsearchOutput::ConnectionFailure, Timeout::Error => e
+    rescue *client.transport.host_unreachable_exceptions, Timeout::Error => e
       @_es = nil
       @_es_info = nil
       if retries < max_retries
@@ -40,7 +43,7 @@ module Fluent::ElasticsearchIndexTemplate
         log.warn "Could not push template(s) to Elasticsearch, resetting connection and trying again. #{e.message}"
         retry
       end
-      raise Fluent::Plugin::ElasticsearchOutput::ConnectionFailure, "Could not push template(s) to Elasticsearch after #{retries} retries. #{e.message}"
+      raise TemplateInstallationFailure, "Could not push template(s) to Elasticsearch after #{retries} retries. #{e.message}"
     end
   end
 

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -22,8 +22,8 @@ end
 
 module Fluent::Plugin
   class ElasticsearchOutput < Output
-    class ConnectionFailure < StandardError; end
-    class ConnectionRetryFailure < Fluent::UnrecoverableError; end
+    class RecoverableRequestFailure < StandardError; end
+    class UnrecoverableRequestFailure < Fluent::UnrecoverableError; end
 
     # MissingIdFieldError is raised for records that do not
     # include the field for the unique record identifier
@@ -264,6 +264,10 @@ EOC
           end
         end
       end
+
+      if @buffer_config.flush_thread_count < 2
+        log.warn "To prevent events traffic jam, you should specify 2 or more 'flush_thread_count'."
+      end
     end
 
     def backend_options
@@ -342,7 +346,6 @@ EOC
                                                                               reload_connections: local_reload_connections,
                                                                               reload_on_failure: @reload_on_failure,
                                                                               resurrect_after: @resurrect_after,
-                                                                              retry_on_failure: 5,
                                                                               logger: @transport_logger,
                                                                               transport_options: {
                                                                                 headers: { 'Content-Type' => @content_type.to_s },
@@ -356,16 +359,7 @@ EOC
                                                                               sniffer_class: @sniffer_class,
                                                                               serializer_class: @serializer_class,
                                                                             }), &adapter_conf)
-        es = Elasticsearch::Client.new transport: transport
-
-        begin
-          raise ConnectionFailure, "Can not reach Elasticsearch cluster (#{connection_options_description})!" unless es.ping
-        rescue *es.transport.host_unreachable_exceptions => e
-          raise ConnectionFailure, "Can not reach Elasticsearch cluster (#{connection_options_description})! #{e.message}"
-        end
-
-        log.info "Connection opened to Elasticsearch cluster => #{connection_options_description}"
-        es
+        Elasticsearch::Client.new transport: transport
       end
     end
 
@@ -632,7 +626,6 @@ EOC
     # send_bulk given a specific bulk request, the original tag,
     # chunk, and bulk_message_count
     def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, index)
-      retries = 0
       begin
 
         log.on_trace { log.trace "bulk request: #{data}" }
@@ -646,20 +639,11 @@ EOC
       rescue RetryStreamError => e
         emit_tag = @retry_tag ? @retry_tag : tag
         router.emit_stream(emit_tag, e.retry_stream)
-      rescue *client.transport.host_unreachable_exceptions => e
-        if retries < 2
-          retries += 1
-          @_es = nil
-          @_es_info = nil
-          log.warn "Could not push logs to Elasticsearch, resetting connection and trying again. #{e.message}"
-          sleep 2**retries
-          retry
-        end
-        raise ConnectionRetryFailure, "Could not push logs to Elasticsearch after #{retries} retries. #{e.message}"
-      rescue Exception
+      rescue => e
         @_es = nil if @reconnect_on_error
         @_es_info = nil if @reconnect_on_error
-        raise
+        # FIXME: identify unrecoverable errors and raise UnrecoverableRequestFailure instead
+        raise RecoverableRequestFailure, "could not push logs to Elasticsearch cluster (#{connection_options_description}): #{e.message}"
       end
     end
   end

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -50,7 +50,6 @@ module Fluent::Plugin
                                                                               reload_connections: @reload_connections,
                                                                               reload_on_failure: @reload_on_failure,
                                                                               resurrect_after: @resurrect_after,
-                                                                              retry_on_failure: 5,
                                                                               logger: @transport_logger,
                                                                               transport_options: {
                                                                                 headers: { 'Content-Type' => @content_type.to_s },
@@ -62,16 +61,7 @@ module Fluent::Plugin
                                                                                 password: @password
                                                                               }
                                                                             }), &adapter_conf)
-        es = Elasticsearch::Client.new transport: transport
-
-        begin
-          raise ConnectionFailure, "Can not reach Elasticsearch cluster (#{connection_options_description(host)})!" unless es.ping
-        rescue *es.transport.host_unreachable_exceptions => e
-          raise ConnectionFailure, "Can not reach Elasticsearch cluster (#{connection_options_description(host)})! #{e.message}"
-        end
-
-        log.info "Connection opened to Elasticsearch cluster => #{connection_options_description(host)}"
-        es
+        Elasticsearch::Client.new transport: transport
       end
     end
 
@@ -220,24 +210,15 @@ module Fluent::Plugin
     end
 
     def send_bulk(data, host, index)
-      retries = 0
       begin
         response = client(host).bulk body: data, index: index
         if response['errors']
           log.error "Could not push log to Elasticsearch: #{response}"
         end
-      rescue *client(host).transport.host_unreachable_exceptions => e
-        if retries < 2
-          retries += 1
-          @_es = nil
-          log.warn "Could not push logs to Elasticsearch, resetting connection and trying again. #{e.message}"
-          sleep 2**retries
-          retry
-        end
-        raise ConnectionRetryFailure, "Could not push logs to Elasticsearch after #{retries} retries. #{e.message}"
-      rescue Exception
+      rescue => e
         @_es = nil if @reconnect_on_error
-        raise
+        # FIXME: identify unrecoverable errors and raise UnrecoverableRequestFailure instead
+        raise RecoverableRequestFailure, "could not push logs to Elasticsearch cluster (#{connection_options_description(host)}): #{e.message}"
       end
     end
 

data/test/plugin/test_out_elasticsearch.rb

@@ -59,10 +59,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
     }
   end
 
-  def stub_elastic_ping(url="http://localhost:9200")
-    stub_request(:head, url).to_return(:status => 200, :body => "", :headers => {})
-  end
-
   def stub_elastic_info(url="http://localhost:9200/", version="6.4.2")
     body ="{\"version\":{\"number\":\"#{version}\"}}"
     stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
@@ -333,6 +329,27 @@ class ElasticsearchOutput < Test::Unit::TestCase
     end
   end
 
+  sub_test_case 'connection exceptions' do
+    test 'default connection exception' do
+      driver(Fluent::Config::Element.new(
+               'ROOT', '', {
+                 '@type' => 'elasticsearch',
+                 'host' => 'log.google.com',
+                 'port' => 777,
+                 'scheme' => 'https',
+                 'path' => '/es/',
+                 'user' => 'john',
+                 'pasword' => 'doe',
+               }, [
+                 Fluent::Config::Element.new('buffer', 'tag', {
+                                             }, [])
+               ]
+             ))
+      logs = driver.logs
+      assert_logs_include(logs, /you should specify 2 or more 'flush_thread_count'/, 1)
+    end
+  end
+
   def test_template_already_present
     config = %{
       host            logs.google.com
@@ -613,20 +630,13 @@ class ElasticsearchOutput < Test::Unit::TestCase
     }
 
     connection_resets = 0
-    # connection start
-    stub_request(:head, "https://john:doe@logs.google.com:778/es//").with do |req|
-      connection_resets += 1
-    end
     # check if template exists
     stub_request(:get, "https://john:doe@logs.google.com:778/es//_template/logstash").with do |req|
-      raise Fluent::Plugin::ElasticsearchOutput::ConnectionFailure, "Test message"
-    end
-    # creation
-    stub_request(:put, "https://john:doe@logs.google.com:778/es//_template/logstash").with do |req|
-      raise Fluent::Plugin::ElasticsearchOutput::ConnectionFailure, "Test message"
+      connection_resets += 1
+      raise Faraday::ConnectionFailed, "Test message"
     end
 
-    assert_raise(Fluent::Plugin::ElasticsearchOutput::ConnectionFailure) do
+    assert_raise(Fluent::ElasticsearchIndexTemplate::TemplateInstallationFailure) do
       driver(config)
     end
 
@@ -913,7 +923,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_write_message_with_bad_chunk
     driver.configure("target_index_key bad_value\n@log_level debug\n")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed({'bad_value'=>"\255"})
@@ -924,7 +933,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def test_writes_to_default_index
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -933,7 +941,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def test_writes_to_default_type
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -943,7 +950,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_writes_to_speficied_index
     driver.configure("index_name myindex\n")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -954,7 +960,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
   class IndexNamePlaceholdersTest < self
     def test_writes_to_speficied_index_with_tag_placeholder
       driver.configure("index_name myindex.${tag}\n")
-      stub_elastic_ping
       stub_elastic
       driver.run(default_tag: 'test') do
         driver.feed(sample_record)
@@ -974,7 +979,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
                                                        }, [])
                          ]
                        ))
-      stub_elastic_ping
       stub_elastic
       time = Time.parse Date.today.iso8601
       driver.run(default_tag: 'test') do
@@ -995,7 +999,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
       time = Time.parse Date.today.iso8601
       pipeline_id = "mypipeline"
       logstash_index = "myindex.#{pipeline_id}"
-      stub_elastic_ping
       stub_elastic
       driver.run(default_tag: 'test') do
         driver.feed(time.to_i, sample_record.merge({"pipeline_id" => pipeline_id}))
@@ -1006,7 +1009,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_writes_to_speficied_index_uppercase
     driver.configure("index_name MyIndex\n")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -1018,7 +1020,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_writes_to_target_index_key
     driver.configure("target_index_key @target_index\n")
-    stub_elastic_ping
     stub_elastic
     record = sample_record.clone
     driver.run(default_tag: 'test') do
@@ -1032,7 +1033,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("target_index_key @target_index
                       logstash_format true")
     time = Time.parse Date.today.iso8601
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(time.to_i, sample_record.merge('@target_index' => 'local-override'))
@@ -1044,7 +1044,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("target_index_key @target_index
                       logstash_format true")
     time = Time.parse Date.today.iso8601
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(time.to_i, sample_record.merge('@target_index' => 'local-override'))
@@ -1057,7 +1056,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
   def test_writes_to_default_index_with_pipeline
     pipeline = "fluentd"
     driver.configure("pipeline #{pipeline}")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -1067,7 +1065,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_writes_to_target_index_key_fallack
     driver.configure("target_index_key @target_index\n")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -1080,7 +1077,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
                       logstash_format true")
     time = Time.parse Date.today.iso8601
     logstash_index = "logstash-#{time.getutc.strftime("%Y.%m.%d")}"
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(time.to_i, sample_record)
@@ -1093,7 +1089,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
       )
   def test_writes_to_speficied_type(data)
     driver('', data["es_version"]).configure("type_name mytype\n")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -1106,7 +1101,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
       )
   def test_writes_to_speficied_type_with_placeholders(data)
     driver('', data["es_version"]).configure("type_name mytype.${tag}\n")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -1121,7 +1115,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
       )
   def test_writes_to_target_type_key(data)
     driver('', data["es_version"]).configure("target_type_key @target_type\n")
-    stub_elastic_ping
     stub_elastic
     record = sample_record.clone
     driver.run(default_tag: 'test') do
@@ -1133,7 +1126,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_writes_to_target_type_key_fallack_to_default
     driver.configure("target_type_key @target_type\n")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -1144,7 +1136,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
   def test_writes_to_target_type_key_fallack_to_type_name
     driver.configure("target_type_key @target_type
                       type_name mytype")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -1159,7 +1150,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
       )
   def test_writes_to_target_type_key_nested(data)
     driver('', data["es_version"]).configure("target_type_key kubernetes.labels.log_type\n")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge('kubernetes' => {
@@ -1174,7 +1164,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_writes_to_target_type_key_fallack_to_default_nested
     driver.configure("target_type_key kubernetes.labels.log_type\n")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge('kubernetes' => {
@@ -1188,7 +1177,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_writes_to_speficied_host
     driver.configure("host 192.168.33.50\n")
-    stub_elastic_ping("http://192.168.33.50:9200")
     elastic_request = stub_elastic("http://192.168.33.50:9200/_bulk")
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -1198,7 +1186,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_writes_to_speficied_port
     driver.configure("port 9201\n")
-    stub_elastic_ping("http://localhost:9201")
     elastic_request = stub_elastic("http://localhost:9201/_bulk")
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -1214,7 +1201,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
     hosts.each do |host_info|
       host, port = host_info
-      stub_elastic_ping("http://#{host}:#{port}")
       stub_elastic_with_store_index_command_counts("http://#{host}:#{port}/_bulk")
     end
 
@@ -1251,7 +1237,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
       {"age" => "30", "height" => "2ft"}
     ]}
 
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(original_hash)
@@ -1265,7 +1250,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
     original_hash =  {"foo" => {"bar" => "baz"}}
     expected_output = {"foo" => {"bar" => "baz"}}
 
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(original_hash)
@@ -1274,7 +1258,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def test_makes_bulk_request
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -1284,7 +1267,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def test_all_records_are_preserved_in_bulk
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -1300,7 +1282,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
     # This is 1 second past midnight in BST, so the UTC index should be the day before
     dt = DateTime.new(2015, 6, 1, 0, 0, 1, "+01:00")
     logstash_index = "logstash-2015.05.31"
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(dt.to_time.to_i, sample_record)
@@ -1315,7 +1296,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
     # ingestion time
     time = Date.today.to_time
     index = "logstash-#{time.strftime("%Y.%m.%d")}"
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(time.to_i, sample_record)
@@ -1328,7 +1308,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
                       logstash_prefix myprefix")
     time = Time.parse Date.today.iso8601
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(time.to_i, sample_record)
@@ -1343,7 +1322,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
                       logstash_prefix myprefix")
     time = Time.parse Date.today.iso8601
     logstash_index = "myprefix#{separator}#{time.getutc.strftime("%Y.%m.%d")}"
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(time.to_i, sample_record)
@@ -1357,7 +1335,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
                       logstash_prefix myprefix-${tag}")
       time = Time.parse Date.today.iso8601
       logstash_index = "myprefix-test-#{time.getutc.strftime("%Y.%m.%d")}"
-      stub_elastic_ping
       stub_elastic
       driver.run(default_tag: 'test') do
         driver.feed(time.to_i, sample_record)
@@ -1380,7 +1357,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
                        ))
       time = Time.parse Date.today.iso8601
       logstash_index = "myprefix-#{time.getutc.strftime("%H")}-#{time.getutc.strftime("%Y.%m.%d")}"
-      stub_elastic_ping
       stub_elastic
       driver.run(default_tag: 'test') do
         driver.feed(time.to_i, sample_record)
@@ -1401,7 +1377,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
       time = Time.parse Date.today.iso8601
       pipeline_id = "mypipeline"
       logstash_index = "myprefix-#{pipeline_id}-#{time.getutc.strftime("%Y.%m.%d")}"
-      stub_elastic_ping
       stub_elastic
       driver.run(default_tag: 'test') do
         driver.feed(time.to_i, sample_record.merge({"pipeline_id" => pipeline_id}))
@@ -1415,7 +1390,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
                       logstash_prefix MyPrefix")
     time = Time.parse Date.today.iso8601
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(time.to_i, sample_record)
@@ -1430,7 +1404,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
                       logstash_dateformat %Y.%m")
     time = Time.parse Date.today.iso8601
     logstash_index = "logstash-#{time.getutc.strftime("%Y.%m")}"
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(time.to_i, sample_record)
@@ -1444,7 +1417,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
                       logstash_dateformat %Y.%m")
     time = Time.parse Date.today.iso8601
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m")}"
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(time.to_i, sample_record)
@@ -1471,7 +1443,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def test_doesnt_add_logstash_timestamp_by_default
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -1481,7 +1452,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_adds_timestamp_when_logstash
     driver.configure("logstash_format true\n")
-    stub_elastic_ping
     stub_elastic
     ts = DateTime.now
     time = Fluent::EventTime.from_time(ts.to_time)
@@ -1494,7 +1464,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_adds_timestamp_when_include_timestamp
     driver.configure("include_timestamp true\n")
-    stub_elastic_ping
     stub_elastic
     ts = DateTime.now
     time = Fluent::EventTime.from_time(ts.to_time)
@@ -1507,7 +1476,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_uses_custom_timestamp_when_included_in_record
     driver.configure("logstash_format true\n")
-    stub_elastic_ping
     stub_elastic
     ts = DateTime.new(2001,2,3).iso8601
     driver.run(default_tag: 'test') do
@@ -1519,7 +1487,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_uses_custom_timestamp_when_included_in_record_without_logstash
     driver.configure("include_timestamp true\n")
-    stub_elastic_ping
     stub_elastic
     ts = DateTime.new(2001,2,3).iso8601
     driver.run(default_tag: 'test') do
@@ -1532,7 +1499,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
   def test_uses_custom_time_key
     driver.configure("logstash_format true
                       time_key vtm\n")
-    stub_elastic_ping
     stub_elastic
     ts = DateTime.new(2001,2,3).iso8601(9)
     driver.run(default_tag: 'test') do
@@ -1546,7 +1512,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("logstash_format true
                       time_key_format %Y-%m-%d %H:%M:%S.%N%z
                       time_key vtm\n")
-    stub_elastic_ping
     stub_elastic
     ts = "2001-02-03 13:14:01.673+02:00"
     driver.run(default_tag: 'test') do
@@ -1562,7 +1527,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
                       index_name test
                       time_key_format %Y-%m-%d %H:%M:%S.%N%z
                       time_key vtm\n")
-    stub_elastic_ping
     stub_elastic
     ts = "2001-02-03 13:14:01.673+02:00"
     driver.run(default_tag: 'test') do
@@ -1577,7 +1541,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("logstash_format true
                       time_key vtm
                       time_key_exclude_timestamp true\n")
-    stub_elastic_ping
     stub_elastic
     ts = DateTime.new(2001,2,3).iso8601
     driver.run(default_tag: 'test') do
@@ -1589,7 +1552,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
   def test_uses_custom_time_key_format
     driver.configure("logstash_format true
                       time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n")
-    stub_elastic_ping
     stub_elastic
     ts = "2001-02-03T13:14:01.673+02:00"
     driver.run(default_tag: 'test') do
@@ -1604,7 +1566,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("include_timestamp true
                       index_name test
                       time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n")
-    stub_elastic_ping
     stub_elastic
     ts = "2001-02-03T13:14:01.673+02:00"
     driver.run(default_tag: 'test') do
@@ -1622,7 +1583,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
     tag_for_error = 'Fluent::ElasticsearchOutput::TimeParser.error' if tag_for_error.nil?
     driver.configure("logstash_format true
                       time_key_format %Y-%m-%dT%H:%M:%S.%N%z\n#{tag_config}\n")
-    stub_elastic_ping
     stub_elastic
 
     ts = "2001/02/03 13:14:01,673+02:00"
@@ -1643,7 +1603,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
   def test_uses_custom_time_key_format_obscure_format
     driver.configure("logstash_format true
                       time_key_format %a %b %d %H:%M:%S %Z %Y\n")
-    stub_elastic_ping
     stub_elastic
     ts = "Thu Nov 29 14:33:20 GMT 2001"
     driver.run(default_tag: 'test') do
@@ -1656,7 +1615,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_uses_nanosecond_precision_by_default
     driver.configure("logstash_format true\n")
-    stub_elastic_ping
     stub_elastic
     time = Fluent::EventTime.new(Time.now.to_i, 123456789)
     driver.run(default_tag: 'test') do
@@ -1669,7 +1627,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
   def test_uses_subsecond_precision_when_configured
     driver.configure("logstash_format true
                       time_precision 3\n")
-    stub_elastic_ping
     stub_elastic
     time = Fluent::EventTime.new(Time.now.to_i, 123456789)
     driver.run(default_tag: 'test') do
@@ -1680,7 +1637,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def test_doesnt_add_tag_key_by_default
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -1690,7 +1646,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_adds_tag_key_when_configured
     driver.configure("include_tag_key true\n")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'mytag') do
       driver.feed(sample_record)
@@ -1701,7 +1656,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_adds_id_key_when_configured
     driver.configure("id_key request_id\n")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -1712,7 +1666,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
   class NestedIdKeyTest < self
     def test_adds_nested_id_key_with_dot
       driver.configure("id_key nested.request_id\n")
-      stub_elastic_ping
       stub_elastic
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
@@ -1722,7 +1675,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
     def test_adds_nested_id_key_with_dollar_dot
       driver.configure("id_key $.nested.request_id\n")
-      stub_elastic_ping
       stub_elastic
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
@@ -1732,7 +1684,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
     def test_adds_nested_id_key_with_bracket
       driver.configure("id_key $['nested']['request_id']\n")
-      stub_elastic_ping
       stub_elastic
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
@@ -1743,7 +1694,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_doesnt_add_id_key_if_missing_when_configured
     driver.configure("id_key another_request_id\n")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -1752,7 +1702,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def test_adds_id_key_when_not_configured
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -1762,7 +1711,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_adds_parent_key_when_configured
     driver.configure("parent_key parent_id\n")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -1773,7 +1721,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
   class NestedParentKeyTest < self
     def test_adds_nested_parent_key_with_dot
       driver.configure("parent_key nested.parent_id\n")
-      stub_elastic_ping
       stub_elastic
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
@@ -1783,7 +1730,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
     def test_adds_nested_parent_key_with_dollar_dot
       driver.configure("parent_key $.nested.parent_id\n")
-      stub_elastic_ping
       stub_elastic
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
@@ -1793,7 +1739,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
     def test_adds_nested_parent_key_with_bracket
       driver.configure("parent_key $['nested']['parent_id']\n")
-      stub_elastic_ping
       stub_elastic
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
@@ -1804,7 +1749,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_doesnt_add_parent_key_if_missing_when_configured
     driver.configure("parent_key another_parent_id\n")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -1813,7 +1757,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def test_adds_parent_key_when_not_configured
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -1823,7 +1766,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_adds_routing_key_when_configured
     driver.configure("routing_key routing_id\n")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -1834,7 +1776,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
   class NestedRoutingKeyTest < self
     def test_adds_nested_routing_key_with_dot
       driver.configure("routing_key nested.routing_id\n")
-      stub_elastic_ping
       stub_elastic
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
@@ -1844,7 +1785,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
     def test_adds_nested_routing_key_with_dollar_dot
       driver.configure("routing_key $.nested.routing_id\n")
-      stub_elastic_ping
       stub_elastic
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
@@ -1854,7 +1794,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
     def test_adds_nested_routing_key_with_bracket
       driver.configure("routing_key $['nested']['routing_id']\n")
-      stub_elastic_ping
       stub_elastic
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
@@ -1865,7 +1804,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_doesnt_add_routing_key_if_missing_when_configured
     driver.configure("routing_key another_routing_id\n")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -1874,7 +1812,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def test_adds_routing_key_when_not_configured
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -1884,7 +1821,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_remove_one_key
     driver.configure("remove_keys key1\n")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge('key1' => 'v1', 'key2' => 'v2'))
@@ -1895,7 +1831,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_remove_multi_keys
     driver.configure("remove_keys key1, key2\n")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge('key1' => 'v1', 'key2' => 'v2'))
@@ -1905,17 +1840,16 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def test_request_error
-    stub_elastic_ping
     stub_elastic_unavailable
-    assert_raise(Elasticsearch::Transport::Transport::Errors::ServiceUnavailable) {
-      driver.run(default_tag: 'test') do
+    assert_raise(Fluent::Plugin::ElasticsearchOutput::RecoverableRequestFailure) {
+      driver.run(default_tag: 'test', shutdown: false) do
         driver.feed(sample_record)
       end
     }
   end
 
   def test_request_forever
-    stub_elastic_ping
+    omit("retry_forever test is unstable.") if ENV["CI"]
     stub_elastic
     driver.configure(Fluent::Config::Element.new(
                'ROOT', '', {
@@ -1927,42 +1861,40 @@ class ElasticsearchOutput < Test::Unit::TestCase
                ]
              ))
     stub_elastic_timeout
-    driver.run(default_tag: 'test', timeout: 10) do
-      driver.feed(sample_record)
-    end
+    assert_raise(Timeout::Error) {
+      driver.run(default_tag: 'test', timeout: 10, force_flush_retry: true) do
+        driver.feed(sample_record)
+      end
+    }
   end
 
-  def test_connection_failed_retry
+  def test_connection_failed
     connection_resets = 0
 
-    stub_elastic_ping(url="http://localhost:9200").with do |req|
-      connection_resets += 1
-    end
-
     stub_request(:post, "http://localhost:9200/_bulk").with do |req|
+      connection_resets += 1
       raise Faraday::ConnectionFailed, "Test message"
     end
 
-    driver.run(default_tag: 'test') do
-      driver.feed(sample_record)
-    end
-    assert_equal(connection_resets, 3)
+    assert_raise(Fluent::Plugin::ElasticsearchOutput::RecoverableRequestFailure) {
+      driver.run(default_tag: 'test', shutdown: false) do
+        driver.feed(sample_record)
+      end
+    }
+    assert_equal(connection_resets, 1)
   end
 
   def test_reconnect_on_error_enabled
     connection_resets = 0
 
-    stub_elastic_ping(url="http://localhost:9200").with do |req|
-      connection_resets += 1
-    end
-
     stub_request(:post, "http://localhost:9200/_bulk").with do |req|
+      connection_resets += 1
       raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
     end
 
     driver.configure("reconnect_on_error true\n")
 
-    assert_raise(ZeroDivisionError) {
+    assert_raise(Fluent::Plugin::ElasticsearchOutput::RecoverableRequestFailure) {
       driver.run(default_tag: 'test', shutdown: false) do
         driver.feed(sample_record)
       end
@@ -1981,17 +1913,14 @@ class ElasticsearchOutput < Test::Unit::TestCase
   def test_reconnect_on_error_disabled
     connection_resets = 0
 
-    stub_elastic_ping(url="http://localhost:9200").with do |req|
-      connection_resets += 1
-    end
-
     stub_request(:post, "http://localhost:9200/_bulk").with do |req|
+      connection_resets += 1
       raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
     end
 
     driver.configure("reconnect_on_error false\n")
 
-    assert_raise(ZeroDivisionError) {
+    assert_raise(Fluent::Plugin::ElasticsearchOutput::RecoverableRequestFailure) {
       driver.run(default_tag: 'test', shutdown: false) do
         driver.feed(sample_record)
       end
@@ -2007,7 +1936,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_bulk_error_retags_when_configured
     driver.configure("retry_tag retry\n")
-    stub_elastic_ping
     stub_request(:post, 'http://localhost:9200/_bulk')
         .to_return(lambda do |req|
       { :status => 200,
@@ -2042,7 +1970,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_create_should_write_records_with_ids_and_skip_those_without
     driver.configure("write_operation create\nid_key my_id\n@log_level debug")
-    stub_elastic_ping
     stub_request(:post, 'http://localhost:9200/_bulk')
         .to_return(lambda do |req|
       { :status => 200,
@@ -2092,7 +2019,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_create_should_write_records_with_ids_and_emit_those_without
     driver.configure("write_operation create\nid_key my_id\nemit_error_for_missing_id true\n@log_level debug")
-    stub_elastic_ping
     stub_request(:post, 'http://localhost:9200/_bulk')
         .to_return(lambda do |req|
       { :status => 200,
@@ -2141,7 +2067,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def test_bulk_error
-    stub_elastic_ping
     stub_request(:post, 'http://localhost:9200/_bulk')
         .to_return(lambda do |req|
       { :status => 200,
@@ -2210,7 +2135,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_update_should_not_write_if_theres_no_id
     driver.configure("write_operation update\n")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -2220,7 +2144,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_upsert_should_not_write_if_theres_no_id
     driver.configure("write_operation upsert\n")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -2230,7 +2153,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_create_should_not_write_if_theres_no_id
     driver.configure("write_operation create\n")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -2241,7 +2163,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
   def test_update_should_write_update_op_and_doc_as_upsert_is_false
     driver.configure("write_operation update
                       id_key request_id")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -2255,7 +2176,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("write_operation update
                       id_key request_id
                       remove_keys_on_update parent_id")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -2267,7 +2187,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
   def test_upsert_should_write_update_op_and_doc_as_upsert_is_true
     driver.configure("write_operation upsert
                       id_key request_id")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -2281,7 +2200,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("write_operation upsert
                       id_key request_id
                       remove_keys_on_update parent_id")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -2296,7 +2214,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("write_operation upsert
                       id_key request_id
                       remove_keys_on_update parent_id")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -2310,7 +2227,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("write_operation upsert
                       id_key id
                       remove_keys_on_update foo,baz")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed("id" => 1, "foo" => "bar", "baz" => "quix", "zip" => "zam")
@@ -2335,7 +2251,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("write_operation upsert
                       id_key id
                       remove_keys_on_update_key keys_to_skip")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed("id" => 1, "foo" => "bar", "baz" => "quix", "keys_to_skip" => ["baz"])
@@ -2360,7 +2275,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
                       id_key id
                       remove_keys_on_update foo,bar
                       remove_keys_on_update_key keys_to_skip")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed("id" => 1, "foo" => "bar", "baz" => "quix", "keys_to_skip" => ["baz"])
@@ -2385,7 +2299,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
   def test_create_should_write_create_op
     driver.configure("write_operation create
                       id_key request_id")
-    stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
@@ -2394,7 +2307,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
   end
 
   def test_include_index_in_url
-    stub_elastic_ping
     stub_elastic('http://localhost:9200/logstash-2018.01.01/_bulk')
 
     driver.configure("index_name logstash-2018.01.01
@@ -2409,7 +2321,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def test_use_simple_sniffer
     require 'fluent/plugin/elasticsearch_simple_sniffer'
-    stub_elastic_ping
     stub_elastic_info
     stub_elastic
     config = %[
@@ -2425,8 +2336,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
     end
     log = driver.logs
     # 2 or 3 - one for the ping, one for the _bulk, (and client.info)
-    assert_logs_include_compare_size(4, ">", log, /In Fluent::Plugin::ElasticsearchSimpleSniffer hosts/)
-    assert_logs_include_compare_size(2, "<=", log, /In Fluent::Plugin::ElasticsearchSimpleSniffer hosts/)
+    assert_logs_include_compare_size(3, ">", log, /In Fluent::Plugin::ElasticsearchSimpleSniffer hosts/)
+    assert_logs_include_compare_size(1, "<=", log, /In Fluent::Plugin::ElasticsearchSimpleSniffer hosts/)
   end
 
 end