fluent-plugin-elasticsearch 5.2.0 → 5.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4c4d872817c5c5f788b40c89adfddec72dc5765666ab90046a38d5e881ba4f2d
-  data.tar.gz: 8e83854fa3eea144ba97e44ba52885dd5f28ef3801fe293c69cf5158e7839789
+  metadata.gz: 7e2737dc1ced4c50a3db85d71c83351e77a606d581fbd768261ca6613b1700f8
+  data.tar.gz: 49a84ff1ea184c4afd43e69fb6cbb89559926bbd6ac063196bd777281be17d2b
 SHA512:
-  metadata.gz: 65eb51a5cd710eb39c54f822f3748bb6ba202ebc87d9aab563adf4befec9696279219f452cc99a1d7528e4c484e03807f195cba8695ec16cc0b3525793a872be
-  data.tar.gz: 3ce726708a2b1900f27999518a56bbf857c28337ce1e3d82ab0acd8fbdb58d92488ddb7e9ccf5a8482768b2c4da1c261b96835f02237321b1a941c6d3b0eb2dc
+  metadata.gz: 157dbe3ab067ec279f2051a8b6c6ac25538821903b48e96a09687b547270796d7f6972d0707225c99cd36694e8a8668d32dc9ebba3599d24f08c814712c0849a
+  data.tar.gz: 73c611531aa95d5d03d8bd5146ab5f6812c30b3be634f5d1ca139b2ffc6206ccebb8915c4fa1e78d2d53706f6c14b7203342d235bd217a89ac9bf0135e3fec21

data/.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+- package-ecosystem: "github-actions"
+  directory: "/"
+  schedule:
+      interval: "weekly"

data/.github/workflows/linux.yml

@@ -2,6 +2,9 @@ name: Testing on Ubuntu
 on:
   - push
   - pull_request
+permissions:
+  contents: read
+
 jobs:
   build:
     runs-on: ${{ matrix.os }}
@@ -13,7 +16,7 @@ jobs:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}

data/.github/workflows/macos.yml

@@ -2,6 +2,9 @@ name: Testing on macOS
 on:
   - push
   - pull_request
+permissions:
+  contents: read
+
 jobs:
   build:
     runs-on: ${{ matrix.os }}
@@ -13,7 +16,7 @@ jobs:
           - macOS-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}

data/.github/workflows/windows.yml

@@ -2,6 +2,9 @@ name: Testing on Windows
 on:
   - push
   - pull_request
+permissions:
+  contents: read
+
 jobs:
   build:
     runs-on: ${{ matrix.os }}
@@ -13,7 +16,7 @@ jobs:
           - windows-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}

data/History.md

@@ -2,6 +2,22 @@
 
 ### [Unreleased]
 
+### 5.2.3
+- Bump actions/checkout from 2 to 3 (#978)
+- chore: Included githubactions in the dependabot config (#977)
+- chore: Set permissions for GitHub actions (#972)
+- Remove nested msgpack\_each in handle\_error (#970)
+- do not overwrite @timestamp in data stream if it already exists in the record (#968)
+
+### 5.2.2
+-  Add missing top level class markers (#961)
+- Ensure use_record_as_seed for same records (#960)
+
+### 5.2.1
+- respect include\_tag\_key and tag\_key setting when using data streams (#936)
+- Handle unsupported version error (#956)
+- Display deprecated warning on ES dynamic plugin (#955)
+
 ### 5.2.0
 -  Migrate to handle Elasticsearch 8 (#949)
 

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '5.2.0'
+  s.version       = '5.2.3'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_compat.rb

@@ -25,3 +25,6 @@ begin
   ::SELECTOR_CLASS = Elasticsearch::Transport::Transport::Connections::Selector
 rescue LoadError
 end
+unless defined?(::Elasticsearch::UnsupportedProductError)
+  class ::Elasticsearch::UnsupportedProductError < StandardError; end
+end

data/lib/fluent/plugin/elasticsearch_error_handler.rb

@@ -35,7 +35,7 @@ class Fluent::Plugin::ElasticsearchErrorHandler
     end
   end
 
-  def handle_error(response, tag, chunk, bulk_message_count, extracted_values)
+  def handle_error(response, tag, chunk, bulk_message_count, extracted_values, unpacked_msg_arr)
     items = response['items']
     if items.nil? || !items.is_a?(Array)
       raise ElasticsearchVersionMismatch, "The response format was unrecognized: #{response}"
@@ -48,7 +48,11 @@ class Fluent::Plugin::ElasticsearchErrorHandler
     meta = {}
     header = {}
     affinity_target_indices = @plugin.get_affinity_target_indices(chunk)
-    chunk.msgpack_each do |time, rawrecord|
+
+    unpacked_msg_arr.each do |msg|
+      time = msg[:time]
+      rawrecord = msg[:record]
+
       bulk_message = ''
       next unless rawrecord.is_a? Hash
       begin

data/lib/fluent/plugin/filter_elasticsearch_genid.rb

@@ -53,7 +53,7 @@ module Fluent::Plugin
       seed += tag + separator if @include_tag_in_seed
       seed += time.to_s + separator if @include_time_in_seed
       if @use_entire_record
-        record.each {|k,v| seed += "|#{k}|#{v}"}
+        record.keys.sort.each {|k| seed += "|#{k}|#{record[k]}"}
       else
         seed += record_keys.map {|k| record[k]}.join(separator)
       end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -492,7 +492,11 @@ EOC
     end
 
     def detect_es_major_version
-      @_es_info ||= client.info
+      begin
+        @_es_info ||= client.info
+      rescue ::Elasticsearch::UnsupportedProductError => e
+        raise Fluent::ConfigError, "Using Elasticsearch client #{client_library_version} is not compatible for your Elasticsearch server. Please check your using elasticsearch gem version and Elasticsearch server."
+      end
       begin
         unless version = @_es_info.dig("version", "number")
           version = @default_elasticsearch_version
@@ -823,6 +827,7 @@ EOC
       bulk_message = Hash.new { |h,k| h[k] = '' }
       header = {}
       meta = {}
+      unpackedMsgArr = {}
 
       tag = chunk.metadata.tag
       chunk_id = dump_unique_id_hex(chunk.unique_id)
@@ -847,9 +852,13 @@ EOC
                    RequestInfo.new(host, nil, meta["_index".freeze], meta.delete("_alias".freeze))
                  end
 
+          unpackedMsgArr[info] = [] if unpackedMsgArr[info].nil?
+          unpackedMsgArr[info] << {:time => time, :record => record}
+
           if split_request?(bulk_message, info)
             bulk_message.each do |info, msgs|
-              send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info) unless msgs.empty?
+              send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info, unpackedMsgArr[info]) unless msgs.empty?
+              unpackedMsgArr[info].clear
               msgs.clear
               # Clear bulk_message_count for this info.
               bulk_message_count[info] = 0;
@@ -872,7 +881,9 @@ EOC
       end
 
       bulk_message.each do |info, msgs|
-        send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info) unless msgs.empty?
+        send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info, unpackedMsgArr[info]) unless msgs.empty?
+
+        unpackedMsgArr[info].clear
         msgs.clear
       end
     end
@@ -1086,7 +1097,7 @@ EOC
 
     # send_bulk given a specific bulk request, the original tag,
     # chunk, and bulk_message_count
-    def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info)
+    def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info, unpacked_msg_arr)
       _logstash_prefix, _logstash_dateformat, index_name, _type_name, template_name, customize_template, deflector_alias, application_name, _pipeline, ilm_policy_id = extracted_values
       if deflector_alias
         template_installation(deflector_alias, template_name, customize_template, application_name, index_name, ilm_policy_id, info.host)
@@ -1109,7 +1120,7 @@ EOC
 
         if response['errors']
           error = Fluent::Plugin::ElasticsearchErrorHandler.new(self)
-          error.handle_error(response, tag, chunk, bulk_message_count, extracted_values)
+          error.handle_error(response, tag, chunk, bulk_message_count, extracted_values, unpacked_msg_arr)
         end
       rescue RetryStreamError => e
         log.trace "router.emit_stream for retry stream doing..."

data/lib/fluent/plugin/out_elasticsearch_data_stream.rb

@@ -243,8 +243,14 @@ module Fluent::Plugin
       chunk.msgpack_each do |time, record|
         next unless record.is_a? Hash
 
+        if @include_tag_key
+          record[@tag_key] = tag
+        end
+
         begin
-          record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
+          unless record.has_key?("@timestamp")
+            record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
+          end
           bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)
         rescue => e
           router.emit_error_event(tag, time, record, e)

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -28,6 +28,8 @@ module Fluent::Plugin
         @dynamic_config[key] = value.to_s
       }
       # end eval all configs
+
+      log.warn "Elasticsearch dynamic plugin will be deprecated and removed in the future. Please consider to use normal Elasticsearch plugin"
     end
 
     def create_meta_config_map

data/test/plugin/test_elasticsearch_error_handler.rb

@@ -2,6 +2,7 @@ require_relative '../helper'
 require 'fluent/plugin/out_elasticsearch'
 require 'fluent/plugin/elasticsearch_error_handler'
 require 'json'
+require 'msgpack'
 
 class TestElasticsearchErrorHandler < Test::Unit::TestCase
 
@@ -54,6 +55,27 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     end
   end
 
+  class MockMsgpackChunk
+    def initialize(chunk)
+      @chunk = chunk
+      @factory = MessagePack::Factory.new
+      @factory.register_type(Fluent::EventTime::TYPE, Fluent::EventTime)
+    end
+
+    def msgpack_each
+      @factory.unpacker(@chunk).each { |time, record| yield(time, record) }
+    end
+  end
+
+  class MockUnpackedMsg
+    def initialize(records)
+      @records = records
+    end
+    def each
+      @records.each { |item| yield({:time => item[:time], :record => item[:record]}) }
+    end
+  end
+
   def setup
     Fluent::Test.setup
     @log_device = Fluent::Test::DummyLogDevice.new
@@ -98,8 +120,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       ]
      }))
       chunk = MockChunk.new(records)
+      unpacked_msg_arr = MockUnpackedMsg.new(records)
       dummy_extracted_values = []
-      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
       assert_equal(1, @plugin.error_events.size)
       expected_log = "failed to parse"
       exception_message = @plugin.error_events.first[:error].message
@@ -140,8 +163,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       ]
      }))
       chunk = MockChunk.new(records)
+      unpacked_msg_arr = MockUnpackedMsg.new(records)
       dummy_extracted_values = []
-      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
       assert_equal(1, @plugin.error_events.size)
       expected_log = "failed to parse"
       exception_message = @plugin.error_events.first[:error].message
@@ -159,8 +183,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       "items" : [{}]
      }))
     chunk = MockChunk.new(records)
+    unpacked_msg_arr = MockUnpackedMsg.new(records)
     dummy_extracted_values = []
-    @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+    @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
     assert_equal(0, @plugin.error_events.size)
     assert_nil(@plugin.error_events[0])
   end
@@ -181,8 +206,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       ]
      }))
     chunk = MockChunk.new(records)
+    unpacked_msg_arr = MockUnpackedMsg.new(records)
     dummy_extracted_values = []
-    @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+    @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
     assert_equal(1, @plugin.error_events.size)
     assert_true(@plugin.error_events[0][:error].respond_to?(:backtrace))
   end
@@ -204,8 +230,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       ]
      }))
     chunk = MockChunk.new(records)
+    unpacked_msg_arr = MockUnpackedMsg.new(records)
     dummy_extracted_values = []
-    @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+    @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
     assert_equal(1, @plugin.error_events.size)
     assert_true(@plugin.error_events[0][:error].respond_to?(:backtrace))
   end
@@ -230,10 +257,11 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       ]
      }))
 
-      chunk = MockChunk.new(records)
-      dummy_extracted_values = []
+    chunk = MockChunk.new(records)
+    unpacked_msg_arr = MockUnpackedMsg.new(records)
+    dummy_extracted_values = []
     assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError) do
-      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
     end
   end
 
@@ -257,10 +285,11 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       ]
      }))
 
-      chunk = MockChunk.new(records)
-      dummy_extracted_values = []
+    chunk = MockChunk.new(records)
+    unpacked_msg_arr = MockUnpackedMsg.new(records)
+    dummy_extracted_values = []
     assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError) do
-      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
+      @handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
     end
   end
 
@@ -290,8 +319,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     begin
       failed = false
       chunk = MockChunk.new(records)
+      unpacked_msg_arr = MockUnpackedMsg.new(records)
       dummy_extracted_values = []
-      handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
+      handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values, unpacked_msg_arr)
     rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
       failed = true
       records = [].tap do |records|
@@ -312,6 +342,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
     end
     chunk = MockChunk.new(records)
+    unpacked_msg_arr = MockUnpackedMsg.new(records)
 
     response = parse_response(%({
       "took" : 1,
@@ -410,7 +441,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     begin
       failed = false
       dummy_extracted_values = []
-      @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
+      @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values, unpacked_msg_arr)
     rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
       failed = true
       records = [].tap do |records|
@@ -439,6 +470,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
     end
     chunk = MockChunk.new(records)
+    unpacked_msg_arr = MockUnpackedMsg.new(records)
 
     response = parse_response(%({
       "took" : 1,
@@ -526,7 +558,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     begin
       failed = false
       dummy_extracted_values = []
-      @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
+      @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values, unpacked_msg_arr)
     rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
       failed = true
       records = [].tap do |records|
@@ -549,6 +581,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
       records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
     end
     chunk = MockChunk.new(records)
+    unpacked_msg_arr = MockUnpackedMsg.new(records)
 
     response = parse_response(%({
       "took" : 1,
@@ -639,7 +672,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     begin
       failed = false
       dummy_extracted_values = []
-      @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
+      @handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values, unpacked_msg_arr)
     rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
       failed = true
       records = [].tap do |records|
@@ -660,4 +693,61 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
     end
     assert_true failed
   end
+
+  def test_nested_msgpack_each
+    cwd = File.dirname(__FILE__)
+    chunk_path = File.join(cwd, 'mock_chunk.dat')
+    chunk_file = File.open(chunk_path, 'rb', 0644)
+    chunk_file.seek(0, IO::SEEK_SET)
+
+    chunk = MockMsgpackChunk.new(chunk_file)
+
+    unpacked_msg_arr = []
+    msg_count = 0
+    count_to_trigger_error_handle = 0
+    chunk.msgpack_each do |time, record|
+      next unless record.is_a? Hash
+
+      unpacked_msg_arr << {:time => time, :record => record}
+      msg_count += 1
+
+      record.each_key do |k|
+        if k != 'aaa' && k != 'bbb' && k != 'ccc' && k != 'log_path'
+          assert_equal(:impossible, k)
+        end
+      end
+
+      if msg_count % 55 == 0
+        if count_to_trigger_error_handle == 1
+          begin
+            response = {}
+            response['errors'] = true
+            response['items'] = []
+            item = {}
+            item['index'] = {}
+            item['index']['status'] = 429
+            item['index']['error'] = {}
+            item['index']['error']['type'] = "es_rejected_execution_exception"
+            abc = 0
+            while abc < unpacked_msg_arr.length
+              abc += 1
+              response['items'] << item
+            end
+
+            dummy_extracted_values = []
+            @handler.handle_error(response, 'atag', chunk, unpacked_msg_arr.length, dummy_extracted_values, unpacked_msg_arr)
+            assert_equal(0, @plugin.error_events.size)
+            assert_nil(@plugin.error_events[0])
+          rescue => e
+            # capture ElasticsearchRequestAbortError, beacuse es_rejected_execution_exception is unrecoverable.
+          end
+        end
+
+        count_to_trigger_error_handle += 1
+        unpacked_msg_arr.clear
+      end # end if
+    end # end chunk.msgpack_each
+
+    chunk_file.close
+  end
 end

data/test/plugin/test_filter_elasticsearch_genid.rb

@@ -132,10 +132,10 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
   end
 
   class UseEntireRecordAsSeedTest < self
-    data("md5" => ["md5", "MuMU0gHOP1cWvvg/J4aEFg=="],
-         "sha1" => ["sha1", "GZ6Iup9Ywyk5spCWtPQbtZnfK0U="],
-         "sha256" => ["sha256", "O4YN0RiXCUAYeaR97UUULRLxgra/R2dvTV47viir5l4="],
-         "sha512" => ["sha512", "FtbwO1xsLUq0KcO0mj0l80rbwFH5rGE3vL+Vgh90+4R/9j+/Ni/ipwhiOoUcetDxj1r5Vf/92B54La+QTu3eMA=="],)
+    data("md5" => ["md5", "OAod7J0DR9s9/rOQnkeSFw=="],
+         "sha1" => ["sha1", "0CT4aMJ4gxMT3TKaYPCYApiVsq8="],
+         "sha256" => ["sha256", "mbAuKF5can0TTj/JBk71AXtOyoVqw5W5gMPUxx6pxLk="],
+         "sha512" => ["sha512", "f7kz5KVuDy+riENePDzqBjGQfbuRNpRBSQMzT2/6hrljXbYtBy3YFmxB86ofIf3zz4ZBao2QM2W7YvcwbRtK1w=="],)
     def test_record
       hash_type, expected = data
       d = create_driver(%[
@@ -151,10 +151,10 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
                    d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
     end
 
-    data("md5" => ["md5", "GJfpWe8ofiGzn97bc9Gh0Q=="],
-         "sha1" => ["sha1", "AVaK67Tz0bEJ8xNEzjOQ6r9fAu4="],
-         "sha256" => ["sha256", "WIXWAuf/Z94Uw95mudloo2bgjhSsSduQIwkKTQsNFgU="],
-         "sha512" => ["sha512", "yjMGGxy8uc7gCrPgm8W6MzJGLFk0GtUwJ6w/91laf6WNywuvG/7T6kNHLagAV8rSW8xzxmtEfyValBO5scuoKw=="],)
+    data("md5" => ["md5", "Hb0jwxofNQP+ufQTKK1U4g=="],
+         "sha1" => ["sha1", "BakTtlotl/u+yOON6YcViTz6nms="],
+         "sha256" => ["sha256", "eLuTCsFqDlk6PfABNyD39r36+yNIBeDTHyNKfJ8fZQw="],
+         "sha512" => ["sha512", "PhPCNGalM4H4xT19DnCBnpwr56lbvCo8wJGyCiH9dWcyhn1nA5l1diYSZlF2fNiq1+wzMqfGvJILIjgQrlAPcg=="],)
     def test_record_with_tag
       hash_type, expected = data
       d = create_driver(%[
@@ -171,10 +171,10 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
                    d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
     end
 
-    data("md5" => ["md5", "5nQSaJ4F1p9rDFign13Lfg=="],
-         "sha1" => ["sha1", "hyo9+0ZFBpizKl2NShs3C8yQcGw="],
-         "sha256" => ["sha256", "romVsZSIksbqYsOSnUzolZQw76ankcy0DgvDZ3CayTo="],
-         "sha512" => ["sha512", "RPU7K2Pt0iVyvV7p5usqcUIIOmfTajD1aa7pkR9qZ89UARH/lpm6ESY9iwuYJj92lxOUuF5OxlEwvV7uXJ07iA=="],)
+    data("md5" => ["md5", "C8vfhC4kecNCNutFCuC6MA=="],
+         "sha1" => ["sha1", "+YWVqUEL90wpKJRrionUJwNgXHg="],
+         "sha256" => ["sha256", "eSqGZqjnO6Uum/4CNfJaolX49+2XKogiGMHGNHiO91Q="],
+         "sha512" => ["sha512", "iVmuD0D+i/WtBwNza09ZXSIW8Xg8/yrUwK/M/EZaCMjz/x5FyyCiVkb1VVKsgNnJy0SYt4w21dhHewu1aXM6HA=="],)
     def test_record_with_time
       hash_type, expected = data
       d = create_driver(%[
@@ -191,10 +191,10 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
                    d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
     end
 
-    data("md5" => ["md5", "zGQF35KlMUibJAcgkgQDtw=="],
-         "sha1" => ["sha1", "1x9RZO1xEuWps090qq4DUIsU9x8="],
-         "sha256" => ["sha256", "eulMz0eF56lBEf31aIs0OG2TGCH/aoPfZbRqfEOkAwk="],
-         "sha512" => ["sha512", "mIiYATtpdUFEFCIZg1FdKssIs7oWY0gJjhSSbet0ddUmqB+CiQAcAMTmrXO6AVSH0vsMvao/8vtC8AsIPfF1fA=="],)
+    data("md5" => ["md5", "lU7d4EiF+2M1zxWcsmBbjg=="],
+         "sha1" => ["sha1", "nghmz1y3KTEFxalfS2/Oe4n4yfQ="],
+         "sha256" => ["sha256", "d0le9UOnUeuGPF/2yEBRM1YzOYeHtxYOE1UU6JgJrvU="],
+         "sha512" => ["sha512", "n7rhisGHUBne6c4Cs9DRMbPror8O5Y/vYajDqAtOaiUTys/Z1EKBMnZQA0iVNFw7joX33cenBW3Yyccct3xSew=="],)
     def test_record_with_tag_and_time
       hash_type, expected = data
       d = create_driver(%[

data/test/plugin/test_out_elasticsearch.rb

@@ -920,6 +920,67 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     end
   end
 
+  class GetElasticsearchIncompatibleVersionTest < self
+    def create_driver(conf='', client_version="7.14")
+      # For request stub to detect compatibility.
+      @client_version ||= client_version
+      # Ensure original implementation existence.
+      Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
+        def detect_es_major_version
+          begin
+            @_es_info ||= client.info
+          rescue ::Elasticsearch::UnsupportedProductError => e
+            raise Fluent::ConfigError, "Using Elasticsearch client #{@client_version} is not compatible for your Elasticsearch server. Please check your using elasticsearch gem version and Elasticsearch server."
+          end
+          begin
+            unless version = @_es_info.dig("version", "number")
+              version = @default_elasticsearch_version
+            end
+          rescue NoMethodError => e
+            log.warn "#{@_es_info} can not dig version information. Assuming Elasticsearch #{@default_elasticsearch_version}", error: e
+            version = @default_elasticsearch_version
+          end
+          version.to_i
+        end
+      CODE
+      Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
+        def client_library_version
+          #{@client_version}
+        end
+      CODE
+      Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput).configure(conf)
+    end
+
+    def test_incompatible_es_version
+      if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.14.0")
+        omit "This test is not effective before elasticsearch 7.14"
+      end
+      config = %{
+        host            logs.google.com
+        port            778
+        scheme          https
+        path            /es/
+        user            john
+        password        doe
+        verify_es_version_at_startup true
+        max_retry_get_es_version 1
+      }
+
+      connection_resets = 0
+      stub_request(:get, "https://logs.google.com:778/es//").
+        with(basic_auth: ['john', 'doe']) do |req|
+        connection_resets += 1
+        raise ::Elasticsearch::UnsupportedProductError
+      end
+
+      assert_raise(Fluent::ConfigError) do
+        create_driver(config)
+      end
+
+      assert_equal(1, connection_resets)
+    end
+  end
+
   class GetElasticsearchVersionWithFallbackTest < self
     def create_driver(conf='', client_version="\"5.0\"")
       # For request stub to detect compatibility.

data/test/plugin/test_out_elasticsearch_data_stream.rb

@@ -19,7 +19,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     @driver = nil
     log = Fluent::Engine.log
     log.out.logs.slice!(0, log.out.logs.length)
-    @bulk_records = 0
+    @bulk_records = []
   end
 
   def elasticsearch_version
@@ -70,8 +70,13 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     }
   end
 
+  SAMPLE_RECORD_TIMESTAMP = Time.now.iso8601
   def sample_record
-    {'@timestamp' => Time.now.iso8601, 'message' => 'Sample record'}
+    {'@timestamp' => SAMPLE_RECORD_TIMESTAMP, 'message' => 'Sample record'}
+  end
+
+  def sample_record_no_timestamp
+    {'message' => 'Sample record no timestamp'}
   end
 
   RESPONSE_ACKNOWLEDGED = {"acknowledged": true}
@@ -115,6 +120,17 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     stub_request(:get, "#{url}/_index_template/#{name}").to_return(:status => [404, TRANSPORT_CLASS::Transport::Errors::NotFound])
   end
 
+
+  def push_bulk_request(req_body)
+    # bulk data must be pair of OP and records
+    # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
+    # {"@timestamp": ...}
+    ops = req_body.split("\n")
+    @bulk_records += ops.values_at(
+      * ops.each_index.select {|i| i.odd? }
+    ).map{ |i| JSON.parse(i) }
+  end
+
   def stub_nonexistent_template_retry?(name="foo_tpl", url="http://localhost:9200")
     stub_request(:get, "#{url}/_index_template/#{name}").
       to_return({ status: 500, body: 'Internal Server Error' }, { status: 404, body: '{}' })
@@ -125,19 +141,19 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
       # bulk data must be pair of OP and records
       # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
       # {"@timestamp": ...}
-      @bulk_records += req.body.split("\n").size / 2
+      push_bulk_request(req.body)
     end
     stub_request(:post, "#{url}/#{ilm_name}/_bulk").with do |req|
       # bulk data must be pair of OP and records
       # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
       # {"@timestamp": ...}
-      @bulk_records += req.body.split("\n").size / 2
+      push_bulk_request(req.body)
     end
     stub_request(:post, "#{url}/#{template_name}/_bulk").with do |req|
       # bulk data must be pair of OP and records
       # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
       # {"@timestamp": ...}
-      @bulk_records += req.body.split("\n").size / 2
+      push_bulk_request(req.body)
     end
   end
 
@@ -600,7 +616,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     driver(conf).run(default_tag: 'test') do
       driver.feed(sample_record)
     end
-    assert_equal 1, @bulk_records
+    assert_equal 1, @bulk_records.length
   end
 
   def test_placeholder_params_unset
@@ -619,7 +635,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     driver(conf).run(default_tag: 'test') do
       driver.feed(sample_record)
     end
-    assert_equal 1, @bulk_records
+    assert_equal 1, @bulk_records.length
   end
 
 
@@ -645,7 +661,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     driver(conf).run(default_tag: 'test') do
       driver.feed(sample_record)
     end
-    assert_equal 1, @bulk_records
+    assert_equal 1, @bulk_records.length
   end
 
   def test_custom_record_placeholder
@@ -675,7 +691,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
         driver.feed(record)
       end
     end
-    assert_equal keys.count, @bulk_records
+    assert_equal keys.count, @bulk_records.length
   end
 
   def test_bulk_insert_feed
@@ -693,7 +709,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     driver(conf).run(default_tag: 'test') do
       driver.feed(sample_record)
     end
-    assert_equal 1, @bulk_records
+    assert_equal 1, @bulk_records.length
   end
 
   def test_template_retry_install_fails
@@ -813,4 +829,109 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
       body: '{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}',
       times: 1)
   end
+
+  def test_doesnt_add_tag_key_when_not_configured
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    config = %{
+      data_stream_name           foo
+      data_stream_template_name  foo_tpl
+      data_stream_ilm_name       foo_ilm_policy
+    }
+
+    stub_default
+    stub_bulk_feed
+    driver(config)
+    driver.run(default_tag: 'mytag') do
+      driver.feed(sample_record)
+    end
+
+    assert_equal(1, @bulk_records.length)
+    assert_false(@bulk_records[0].has_key?('tag'))
+  end
+
+
+  def test_adds_tag_key_when_configured
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    config = %{
+      data_stream_name           foo
+      data_stream_template_name  foo_tpl
+      data_stream_ilm_name       foo_ilm_policy
+      include_tag_key            true
+    }
+
+    stub_default
+    stub_bulk_feed
+    driver(config)
+    driver.run(default_tag: 'mytag') do
+      driver.feed(sample_record)
+    end
+
+    assert_equal(1, @bulk_records.length)
+    assert(@bulk_records[0].has_key?('tag'))
+    assert_equal('mytag', @bulk_records[0]['tag'])
+  end
+
+  def test_adds_custom_tag_key_when_configured
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    config = %{
+      data_stream_name           foo
+      data_stream_template_name  foo_tpl
+      data_stream_ilm_name       foo_ilm_policy
+      include_tag_key            true
+      tag_key                    custom_tag_key
+    }
+
+    stub_default
+    stub_bulk_feed
+    driver(config)
+    driver.run(default_tag: 'mytag') do
+      driver.feed(sample_record)
+    end
+
+    assert_equal(1, @bulk_records.length)
+    assert(@bulk_records[0].has_key?('custom_tag_key'))
+    assert_equal('mytag', @bulk_records[0]['custom_tag_key'])
+  end
+
+  def test_use_record_timestamp_if_present
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    stub_default
+    stub_bulk_feed
+    conf = config_element(
+      'ROOT', '', {
+        '@type' => ELASTIC_DATA_STREAM_TYPE,
+        'data_stream_name' => 'foo',
+        'data_stream_ilm_name' => 'foo_ilm_policy',
+        'data_stream_template_name' => 'foo_tpl'
+      })
+    driver(conf).run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+    assert_equal 1, @bulk_records.length
+    assert(@bulk_records[0].has_key?('@timestamp'))
+    assert_equal SAMPLE_RECORD_TIMESTAMP, @bulk_records[0]['@timestamp']
+  end
+
+  def test_add_timestamp_if_not_present_in_record
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    stub_default
+    stub_bulk_feed
+    conf = config_element(
+      'ROOT', '', {
+        '@type' => ELASTIC_DATA_STREAM_TYPE,
+        'data_stream_name' => 'foo',
+        'data_stream_ilm_name' => 'foo_ilm_policy',
+        'data_stream_template_name' => 'foo_tpl'
+      })
+    driver(conf).run(default_tag: 'test') do
+      driver.feed(sample_record_no_timestamp)
+    end
+    assert_equal 1, @bulk_records.length
+    assert(@bulk_records[0].has_key?('@timestamp'))
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 5.2.0
+  version: 5.2.3
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-21 00:00:00.000000000 Z
+date: 2022-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -151,6 +151,7 @@ files:
 - ".editorconfig"
 - ".github/ISSUE_TEMPLATE/bug_report.md"
 - ".github/ISSUE_TEMPLATE/feature_request.md"
+- ".github/dependabot.yml"
 - ".github/workflows/issue-auto-closer.yml"
 - ".github/workflows/linux.yml"
 - ".github/workflows/macos.yml"
@@ -187,6 +188,7 @@ files:
 - lib/fluent/plugin/out_elasticsearch_data_stream.rb
 - lib/fluent/plugin/out_elasticsearch_dynamic.rb
 - test/helper.rb
+- test/plugin/mock_chunk.dat
 - test/plugin/test_alias_template.json
 - test/plugin/test_elasticsearch_error_handler.rb
 - test/plugin/test_elasticsearch_fallback_selector.rb
@@ -222,12 +224,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.30
+rubygems_version: 3.2.32
 signing_key:
 specification_version: 4
 summary: Elasticsearch output plugin for Fluent event collector
 test_files:
 - test/helper.rb
+- test/plugin/mock_chunk.dat
 - test/plugin/test_alias_template.json
 - test/plugin/test_elasticsearch_error_handler.rb
 - test/plugin/test_elasticsearch_fallback_selector.rb