fluent-plugin-elasticsearch 4.1.1 → 5.4.3

data/lib/fluent/plugin/out_elasticsearch_data_stream.rb

@@ -0,0 +1,298 @@
+
+require_relative 'out_elasticsearch'
+
+module Fluent::Plugin
+  class ElasticsearchOutputDataStream < ElasticsearchOutput
+
+    Fluent::Plugin.register_output('elasticsearch_data_stream', self)
+
+    helpers :event_emitter
+
+    config_param :data_stream_name, :string
+    config_param :data_stream_ilm_name, :string, :default => nil
+    config_param :data_stream_template_name, :string, :default => nil
+    config_param :data_stream_ilm_policy, :string, :default => nil
+    config_param :data_stream_ilm_policy_overwrite, :bool, :default => false
+    config_param :data_stream_template_use_index_patterns_wildcard, :bool, :default => true
+
+    # Elasticsearch 7.9 or later always support new style of index template.
+    config_set_default :use_legacy_template, false
+
+    INVALID_START_CHRACTERS = ["-", "_", "+", "."]
+    INVALID_CHARACTERS = ["\\", "/", "*", "?", "\"", "<", ">", "|", " ", ",", "#", ":"]
+
+    def configure(conf)
+      super
+
+      if Gem::Version.new(TRANSPORT_CLASS::VERSION) < Gem::Version.new("8.0.0")
+        begin
+          require 'elasticsearch/api'
+          require 'elasticsearch/xpack'
+        rescue LoadError
+          raise Fluent::ConfigError, "'elasticsearch/api', 'elasticsearch/xpack' are required for <@elasticsearch_data_stream>."
+        end
+      else
+        begin
+          require 'elasticsearch/api'
+        rescue LoadError
+          raise Fluent::ConfigError, "'elasticsearch/api is required for <@elasticsearch_data_stream>."
+        end
+      end
+
+      @data_stream_ilm_name = "#{@data_stream_name}_policy" if @data_stream_ilm_name.nil?
+      @data_stream_template_name = "#{@data_stream_name}_template" if @data_stream_template_name.nil?
+      @data_stream_ilm_policy = File.read(File.join(File.dirname(__FILE__), "default-ilm-policy.json")) if @data_stream_ilm_policy.nil?
+
+      # ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
+      unless placeholder?(:data_stream_name_placeholder, @data_stream_name)
+        validate_data_stream_parameters
+      else
+        @use_placeholder = true
+        @data_stream_names = []
+      end
+
+      unless @use_placeholder
+        begin
+          @data_stream_names = [@data_stream_name]
+          retry_operate(@max_retry_putting_template,
+                        @fail_on_putting_template_retry_exceed,
+                        @catch_transport_exception_on_retry) do
+            create_ilm_policy(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name)
+            create_index_template(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name)
+            create_data_stream(@data_stream_name)
+          end
+        rescue => e
+          raise Fluent::ConfigError, "Failed to create data stream: <#{@data_stream_name}> #{e.message}"
+        end
+      end
+    end
+
+    def validate_data_stream_parameters
+      {"data_stream_name" => @data_stream_name,
+       "data_stream_template_name"=> @data_stream_template_name,
+       "data_stream_ilm_name" => @data_stream_ilm_name}.each do |parameter, value|
+        unless valid_data_stream_parameters?(value)
+          unless start_with_valid_characters?(value)
+            if not_dots?(value)
+              raise Fluent::ConfigError, "'#{parameter}' must not start with #{INVALID_START_CHRACTERS.join(",")}: <#{value}>"
+            else
+              raise Fluent::ConfigError, "'#{parameter}' must not be . or ..: <#{value}>"
+            end
+          end
+          unless valid_characters?(value)
+            raise Fluent::ConfigError, "'#{parameter}' must not contain invalid characters #{INVALID_CHARACTERS.join(",")}: <#{value}>"
+          end
+          unless lowercase_only?(value)
+            raise Fluent::ConfigError, "'#{parameter}' must be lowercase only: <#{value}>"
+          end
+          if value.bytes.size > 255
+            raise Fluent::ConfigError, "'#{parameter}' must not be longer than 255 bytes: <#{value}>"
+          end
+        end
+      end
+    end
+
+    def create_ilm_policy(datastream_name, template_name, ilm_name, host = nil)
+      unless @data_stream_ilm_policy_overwrite
+        return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host) or ilm_policy_exists?(ilm_name, host)
+      end
+
+      params = {
+        body: @data_stream_ilm_policy
+      }
+      retry_operate(@max_retry_putting_template,
+                    @fail_on_putting_template_retry_exceed,
+                    @catch_transport_exception_on_retry) do
+        if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("8.0.0")
+          client(host).ilm.put_lifecycle(params.merge(policy: ilm_name))
+        else
+          client(host).xpack.ilm.put_policy(params.merge(policy_id: ilm_name))
+        end
+      end
+    end
+
+    def create_index_template(datastream_name, template_name, ilm_name, host = nil)
+      return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host)
+      wildcard = @data_stream_template_use_index_patterns_wildcard ? '*' : ''
+      body = {
+        "index_patterns" => ["#{datastream_name}#{wildcard}"],
+        "data_stream" => {},
+        "template" => {
+          "settings" => {
+            "index.lifecycle.name" => "#{ilm_name}"
+          }
+        }
+      }
+      params = {
+        name: template_name,
+        body: body
+      }
+      retry_operate(@max_retry_putting_template,
+                    @fail_on_putting_template_retry_exceed,
+                    @catch_transport_exception_on_retry) do
+        client(host).indices.put_index_template(params)
+      end
+    end
+
+    def data_stream_exist?(datastream_name, host = nil)
+      params = {
+        name: datastream_name
+      }
+      begin
+        response = client(host).indices.get_data_stream(params)
+        return (not response.is_a?(TRANSPORT_CLASS::Transport::Errors::NotFound))
+      rescue TRANSPORT_CLASS::Transport::Errors::NotFound => e
+        log.info "Specified data stream does not exist. Will be created: <#{e}>"
+        return false
+      end
+    end
+
+    def create_data_stream(datastream_name, host = nil)
+      return if data_stream_exist?(datastream_name, host)
+      params = {
+        name: datastream_name
+      }
+      retry_operate(@max_retry_putting_template,
+                    @fail_on_putting_template_retry_exceed,
+                    @catch_transport_exception_on_retry) do
+        client(host).indices.create_data_stream(params)
+      end
+    end
+
+    def ilm_policy_exists?(policy_id, host = nil)
+      begin
+        if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("8.0.0")
+          client(host).ilm.get_lifecycle(policy: policy_id)
+        else
+          client(host).ilm.get_policy(policy_id: policy_id)
+        end
+        true
+      rescue
+        false
+      end
+    end
+
+    def template_exists?(name, host = nil)
+      if @use_legacy_template
+        client(host).indices.get_template(:name => name)
+      else
+        client(host).indices.get_index_template(:name => name)
+      end
+      return true
+    rescue TRANSPORT_CLASS::Transport::Errors::NotFound
+      return false
+    end
+
+    def valid_data_stream_parameters?(data_stream_parameter)
+      lowercase_only?(data_stream_parameter) and
+        valid_characters?(data_stream_parameter) and
+        start_with_valid_characters?(data_stream_parameter) and
+        not_dots?(data_stream_parameter) and
+        data_stream_parameter.bytes.size <= 255
+    end
+
+    def lowercase_only?(data_stream_parameter)
+      data_stream_parameter.downcase == data_stream_parameter
+    end
+
+    def valid_characters?(data_stream_parameter)
+      not (INVALID_CHARACTERS.each.any? do |v| data_stream_parameter.include?(v) end)
+    end
+
+    def start_with_valid_characters?(data_stream_parameter)
+      not (INVALID_START_CHRACTERS.each.any? do |v| data_stream_parameter.start_with?(v) end)
+    end
+
+    def not_dots?(data_stream_parameter)
+      not (data_stream_parameter == "." or data_stream_parameter == "..")
+    end
+
+    def client_library_version
+      Elasticsearch::VERSION
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def write(chunk)
+      data_stream_name = @data_stream_name
+      data_stream_template_name = @data_stream_template_name
+      data_stream_ilm_name = @data_stream_ilm_name
+      host = nil
+      if @use_placeholder
+        host = if @hosts
+                 extract_placeholders(@hosts, chunk)
+               else
+                 extract_placeholders(@host, chunk)
+               end
+        data_stream_name = extract_placeholders(@data_stream_name, chunk)
+        data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk)
+        data_stream_ilm_name = extract_placeholders(@data_stream_ilm_name, chunk)
+        unless @data_stream_names.include?(data_stream_name)
+          begin
+            create_ilm_policy(data_stream_name, data_stream_template_name, data_stream_ilm_name, host)
+            create_index_template(data_stream_name, data_stream_template_name, data_stream_ilm_name, host)
+            create_data_stream(data_stream_name)
+            @data_stream_names << data_stream_name
+          rescue => e
+            raise Fluent::ConfigError, "Failed to create data stream: <#{data_stream_name}> #{e.message}"
+          end
+        end
+      end
+
+      bulk_message = ""
+      headers = {
+        CREATE_OP => {}
+      }
+      tag = chunk.metadata.tag
+      chunk.msgpack_each do |time, record|
+        next unless record.is_a? Hash
+
+        if @include_tag_key
+          record[@tag_key] = tag
+        end
+
+        begin
+          unless record.has_key?("@timestamp")
+            record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
+          end
+          bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)
+        rescue => e
+          router.emit_error_event(tag, time, record, e)
+        end
+      end
+
+      prepared_data = if compression
+        gzip(bulk_message)
+      else
+        bulk_message
+      end
+
+      params = {
+        index: data_stream_name,
+        body: prepared_data
+      }
+      begin
+        response = client(host, compression).bulk(params)
+        if response['errors']
+          log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{response}"
+          @num_errors_metrics.inc
+        end
+      rescue => e
+        raise RecoverableRequestFailure, "could not push logs to Elasticsearch cluster (#{data_stream_name}): #{e.message}"
+      end
+    end
+
+    def append_record_to_messages(op, meta, header, record, msgs)
+      header[CREATE_OP] = meta
+      msgs << @dump_proc.call(header) << BODY_DELIMITER
+      msgs << @dump_proc.call(record) << BODY_DELIMITER
+      msgs
+    end
+
+    def retry_stream_retryable?
+      @buffer.storable?
+    end
+  end
+end

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -28,6 +28,8 @@ module Fluent::Plugin
         @dynamic_config[key] = value.to_s
       }
       # end eval all configs
+
+      log.warn "Elasticsearch dynamic plugin will be deprecated and removed in the future. Please consider to use normal Elasticsearch plugin"
     end
 
     def create_meta_config_map
@@ -53,7 +55,7 @@ module Fluent::Plugin
                        end
         headers = { 'Content-Type' => @content_type.to_s, }.merge(gzip_headers)
         ssl_options = { verify: @ssl_verify, ca_file: @ca_file}.merge(@ssl_version_options)
-        transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
+        transport = TRANSPORT_CLASS::Transport::HTTP::Faraday.new(connection_options.merge(
                                                                             options: {
                                                                               reload_connections: @reload_connections,
                                                                               reload_on_failure: @reload_on_failure,

data/test/helper.rb

@@ -5,9 +5,6 @@ SimpleCov.start do
   end
 end
 
-require 'coveralls'
-Coveralls.wear!
-
 # needs to be after simplecov but before test/unit, because fluentd sets default
 # encoding to ASCII-8BIT, but coverall might load git data which could contain a
 # UTF-8 character
@@ -18,7 +15,6 @@ end
 
 require 'test/unit'
 require 'fluent/test'
-require 'minitest/pride'
 
 require 'webmock/test_unit'
 WebMock.disable_net_connect!

data/test/plugin/test_elasticsearch_error_handler.rb

@@ -2,6 +2,7 @@ require_relative '../helper'
 require 'fluent/plugin/out_elasticsearch'
 require 'fluent/plugin/elasticsearch_error_handler'
 require 'json'
+require 'msgpack'
 
 class TestElasticsearchErrorHandler < Test::Unit::TestCase
 
@@ -27,13 +28,18 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
        @error_events << {:tag => tag, :time=>time, :record=>record, :error=>e}
     end
 
-    def process_message(tag, meta, header, time, record, extracted_values)
+    def process_message(tag, meta, header, time, record, affinity_target_indices, extracted_values)
       return [meta, header, record]
     end
 
+    def get_affinity_target_indices(chunk)
+      indices = Hash.new
+      indices
+    end
+
     def append_record_to_messages(op, meta, header, record, msgs)
       if record.has_key?('raise') && record['raise']
-        raise Exception('process_message')
+        raise 'process_message'
       end
       return true
     end
@@ -49,6 +55,27 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     end
   end
 
+  class MockMsgpackChunk
+    def initialize(chunk)
+      @chunk = chunk
+      @factory = MessagePack::Factory.new
+      @factory.register_type(Fluent::EventTime::TYPE, Fluent::EventTime)
+    end
+
+    def msgpack_each
+      @factory.unpacker(@chunk).each { |time, record| yield(time, record) }
+    end
+  end
+
+  class MockUnpackedMsg
+    def initialize(records)
+      @records = records
+    end
+    def each
+      @records.each { |item| yield({:time => item[:time], :record => item[:record]}) }
+    end
+  end
+
   def setup
     Fluent::Test.setup
     @log_device = Fluent::Test::DummyLogDevice.new
@@ -93,8 +120,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       ]
      }))
       chunk = MockChunk.new(records)
+      unpacked_msg_arr = MockUnpackedMsg.new(records)
       dummy_extracted_values = []
-      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
       assert_equal(1, @plugin.error_events.size)
       expected_log = "failed to parse"
       exception_message = @plugin.error_events.first[:error].message
@@ -135,8 +163,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       ]
      }))
       chunk = MockChunk.new(records)
+      unpacked_msg_arr = MockUnpackedMsg.new(records)
       dummy_extracted_values = []
-      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
       assert_equal(1, @plugin.error_events.size)
       expected_log = "failed to parse"
       exception_message = @plugin.error_events.first[:error].message
@@ -154,8 +183,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       "items" : [{}]
      }))
     chunk = MockChunk.new(records)
+    unpacked_msg_arr = MockUnpackedMsg.new(records)
     dummy_extracted_values = []
-    @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+    @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
     assert_equal(0, @plugin.error_events.size)
     assert_nil(@plugin.error_events[0])
   end
@@ -176,8 +206,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       ]
      }))
     chunk = MockChunk.new(records)
+    unpacked_msg_arr = MockUnpackedMsg.new(records)
     dummy_extracted_values = []
-    @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+    @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
     assert_equal(1, @plugin.error_events.size)
     assert_true(@plugin.error_events[0][:error].respond_to?(:backtrace))
   end
@@ -199,8 +230,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       ]
      }))
     chunk = MockChunk.new(records)
+    unpacked_msg_arr = MockUnpackedMsg.new(records)
     dummy_extracted_values = []
-    @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+    @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
     assert_equal(1, @plugin.error_events.size)
     assert_true(@plugin.error_events[0][:error].respond_to?(:backtrace))
   end
@@ -225,10 +257,11 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       ]
      }))
 
-      chunk = MockChunk.new(records)
-      dummy_extracted_values = []
+    chunk = MockChunk.new(records)
+    unpacked_msg_arr = MockUnpackedMsg.new(records)
+    dummy_extracted_values = []
     assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError) do
-      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
     end
   end
 
@@ -252,10 +285,11 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       ]
      }))
 
-      chunk = MockChunk.new(records)
-      dummy_extracted_values = []
+    chunk = MockChunk.new(records)
+    unpacked_msg_arr = MockUnpackedMsg.new(records)
+    dummy_extracted_values = []
     assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError) do
-      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
     end
   end
 
@@ -285,8 +319,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     begin
       failed = false
       chunk = MockChunk.new(records)
+      unpacked_msg_arr = MockUnpackedMsg.new(records)
       dummy_extracted_values = []
-      handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
+      handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values, unpacked_msg_arr)
     rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
       failed = true
       records = [].tap do |records|
@@ -302,11 +337,12 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
   def test_retry_error
     records = []
     error_records = Hash.new(false)
-    error_records.merge!({0=>true, 4=>true, 9=>true})
+    error_records.merge!({0=>true, 4=>true})
     10.times do |i|
       records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
     end
     chunk = MockChunk.new(records)
+    unpacked_msg_arr = MockUnpackedMsg.new(records)
 
     response = parse_response(%({
       "took" : 1,
@@ -386,6 +422,18 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
               "reason":"unrecognized error"
             }
           }
+        },
+        {
+          "create" : {
+            "_index" : "foo",
+            "_type"  : "bar",
+            "_id" : "9",
+            "status" : 500,
+            "error" : {
+              "type" : "json_parse_exception",
+              "reason":"Invalid UTF-8 start byte 0x92\\n at [Source: org.elasticsearch.transport.netty4.ByteBufStreamInput@204fe9c9; line: 1, column: 81]"
+            }
+          }
         }
       ]
     }))
@@ -393,19 +441,19 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     begin
       failed = false
       dummy_extracted_values = []
-      @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
+      @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values, unpacked_msg_arr)
     rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
       failed = true
       records = [].tap do |records|
         next unless e.respond_to?(:retry_stream)
         e.retry_stream.each {|time, record| records << record}
       end
-      assert_equal 2, records.length
-      assert_equal 2, records[0]['_id']
-      assert_equal 8, records[1]['_id']
+      assert_equal 2, records.length, "Exp. retry_stream to contain records"
+      assert_equal 2, records[0]['_id'], "Exp record with given ID to in retry_stream"
+      assert_equal 8, records[1]['_id'], "Exp record with given ID to in retry_stream"
       error_ids = @plugin.error_events.collect {|h| h[:record]['_id']}
-      assert_equal 3, error_ids.length
-      assert_equal [5, 6, 7], error_ids
+      assert_equal 4, error_ids.length, "Exp. a certain number of records to be dropped from retry_stream"
+      assert_equal [5, 6, 7, 9], error_ids, "Exp. specific records to be dropped from retry_stream"
       @plugin.error_events.collect {|h| h[:error]}.each do |e|
         assert_true e.respond_to?(:backtrace)
       end
@@ -422,6 +470,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
     end
     chunk = MockChunk.new(records)
+    unpacked_msg_arr = MockUnpackedMsg.new(records)
 
     response = parse_response(%({
       "took" : 1,
@@ -509,7 +558,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     begin
       failed = false
       dummy_extracted_values = []
-      @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
+      @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values, unpacked_msg_arr)
     rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
       failed = true
       records = [].tap do |records|
@@ -532,6 +581,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
     end
     chunk = MockChunk.new(records)
+    unpacked_msg_arr = MockUnpackedMsg.new(records)
 
     response = parse_response(%({
       "took" : 1,
@@ -622,7 +672,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     begin
       failed = false
       dummy_extracted_values = []
-      @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
+      @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values, unpacked_msg_arr)
     rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
       failed = true
       records = [].tap do |records|
@@ -643,4 +693,61 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     end
     assert_true failed
   end
+
+  def test_nested_msgpack_each
+    cwd = File.dirname(__FILE__)
+    chunk_path = File.join(cwd, 'mock_chunk.dat')
+    chunk_file = File.open(chunk_path, 'rb', 0644)
+    chunk_file.seek(0, IO::SEEK_SET)
+
+    chunk = MockMsgpackChunk.new(chunk_file)
+
+    unpacked_msg_arr = []
+    msg_count = 0
+    count_to_trigger_error_handle = 0
+    chunk.msgpack_each do |time, record|
+      next unless record.is_a? Hash
+
+      unpacked_msg_arr << {:time => time, :record => record}
+      msg_count += 1
+
+      record.each_key do |k|
+        if k != 'aaa' && k != 'bbb' && k != 'ccc' && k != 'log_path'
+          assert_equal(:impossible, k)
+        end
+      end
+
+      if msg_count % 55 == 0
+        if count_to_trigger_error_handle == 1
+          begin
+            response = {}
+            response['errors'] = true
+            response['items'] = []
+            item = {}
+            item['index'] = {}
+            item['index']['status'] = 429
+            item['index']['error'] = {}
+            item['index']['error']['type'] = "es_rejected_execution_exception"
+            abc = 0
+            while abc < unpacked_msg_arr.length
+              abc += 1
+              response['items'] << item
+            end
+
+            dummy_extracted_values = []
+            @handler.handle_error(response, 'atag', chunk, unpacked_msg_arr.length, dummy_extracted_values, unpacked_msg_arr)
+            assert_equal(0, @plugin.error_events.size)
+            assert_nil(@plugin.error_events[0])
+          rescue => e
+            # capture ElasticsearchRequestAbortError, beacuse es_rejected_execution_exception is unrecoverable.
+          end
+        end
+
+        count_to_trigger_error_handle += 1
+        unpacked_msg_arr.clear
+      end # end if
+    end # end chunk.msgpack_each
+
+    chunk_file.close
+  end
 end

data/test/plugin/test_elasticsearch_fallback_selector.rb

@@ -15,20 +15,28 @@ class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
   def stub_elastic(url="http://localhost:9200/_bulk")
     stub_request(:post, url).with do |req|
       @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
+    end.to_return({:status => 200, :body => "{}", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
+  end
+
+  def elasticsearch_version
+    if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
+      TRANSPORT_CLASS::VERSION
+    else
+      '6.4.2'.freeze
     end
   end
 
-  def stub_elastic_info(url="http://localhost:9200/", version="6.4.2")
-    body ="{\"version\":{\"number\":\"#{version}\"}}"
-    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
+  def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version)
+    body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
+    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
   end
 
-  def stub_elastic_info_not_found(url="http://localhost:9200/", version="6.4.2")
-    stub_request(:get, url).to_return(:status => [404, "Not Found"])
+  def stub_elastic_info_not_found(url="http://localhost:9200/", version=elasticsearch_version)
+    stub_request(:get, url).to_return(:status => [404, "Not Found"], headers: {'x-elastic-product' => 'Elasticsearch' })
   end
 
-  def stub_elastic_info_unavailable(url="http://localhost:9200/", version="6.4.2")
-    stub_request(:get, url).to_return(:status => [503, "Service Unavailable"])
+  def stub_elastic_info_unavailable(url="http://localhost:9200/", version=elasticsearch_version)
+    stub_request(:get, url).to_return(:status => [503, "Service Unavailable"], headers: {'x-elastic-product' => 'Elasticsearch' })
   end
 
   def sample_record(content={})
@@ -58,8 +66,9 @@ class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
       with_transporter_log true
       reload_connections true
       reload_after 10
+      catch_transport_exception_on_retry false # For fallback testing
     ]
-    assert_raise(Elasticsearch::Transport::Transport::Errors::NotFound) do
+    assert_raise(TRANSPORT_CLASS::Transport::Errors::NotFound) do
       driver(config)
     end
     driver.run(default_tag: 'test') do