fluent-plugin-elasticsearch 5.4.2 → 5.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d654e4c24122a1f08b7440556ff43eca647e9aae360c4ebc43f4e29d7a8abfec
-  data.tar.gz: 961f8dab627d2628e854e4b6b64a30fb3719c8387dfd73a67aa6da8f059eb447
+  metadata.gz: 5808f1d08b2a774e67ea22cbf9b678100a1a630c22f6d27ed4c2c656b396f32c
+  data.tar.gz: 836d9697ba68a2b86875429abd31fd09b8d4533d26b2dab4ede5ebdf64a9aad7
 SHA512:
-  metadata.gz: 78d3e7ab001e326c47a192798da97b9d44956db46869b807589818469b8c74cfc36bc936c67f2fdffed3c2f2967d4c9ede1737c8c6d774362f524861eee74186
-  data.tar.gz: 85e1144f3b227d28772cf785932a44bdf6d73307b7cc0401098acd97083b9638089d5a1e5ad2343cc9a7f64f4d93e274db99419fe07fdfe5b785176d667e6592
+  metadata.gz: 3c349e163f308a421d5bc7f74f170def2a1b3845650680b93479a01fcdcd32b2a89dfcb49903ca6c3eaea041beb9dcead4ce5735189f83aa95f9151c2ce3433a
+  data.tar.gz: 0cb7d133409a0289a00fd3e09e9447ac92a3ab72d74cefe92e191477e3bbba0673a322448d8828bb0adc37fcb6f7c523469ec9a95273a5a86fade6fb0dc9cdc8

data/.github/workflows/linux.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.0', '3.1', '3.2' ]
+        ruby: [ '3.0', '3.1', '3.2', '3.3', '3.4' ]
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/macos.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.0', '3.1', '3.2' ]
+        ruby: [ '3.0', '3.1', '3.2', '3.3', '3.4' ]
         os:
           - macOS-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/windows.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.0', '3.1', '3.2' ]
+        ruby: [ '3.0', '3.1', '3.2', '3.3', '3.4' ]
         os:
           - windows-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/History.md

@@ -2,7 +2,16 @@
 
 ### [Unreleased]
 
-### 5.4.1
+### 5.4.4
+- docs: Clarify README docs for template_name and templates (#1043)
+- `out_elasticsearch_data_stream`: Avoid long worrying log line at info level (#1044)
+- `out_elasticsearch_data_stream`: Support `id_key` and `remove_keys` (#1057)
+- `out_elasticsearch`: Fix memory leaks when exception was raised frequently (#1065)
+
+### 5.4.3
+- add data_stream_template\_use\_index\_patterns\_wildcard in elasticsearch\_data\_stream (#1040)
+
+### 5.4.2
 - in\_elasticsearch: Avoid to use deprecated endpoint for clear\_scroll (#1039)
 
 ### 5.4.1

data/README.md

@@ -527,7 +527,7 @@ into same document with data1 as wanted and duplicated document is avoided.
 
 ### template_name
 
-The name of the template to define. If a template by the name given is already present, it will be left unchanged, unless [template_overwrite](#template_overwrite) is set, in which case the template will be updated.
+The name of the index template to create on fluentd startup. If a template by the name given is already present, it will be left unchanged, unless [template_overwrite](#template_overwrite) is set, in which case the template will be updated.
 
 This parameter along with template_file allow the plugin to behave similarly to Logstash (it installs a template at creation time) so that raw records are available. See [https://github.com/uken/fluent-plugin-elasticsearch/issues/33](https://github.com/uken/fluent-plugin-elasticsearch/issues/33).
 
@@ -541,7 +541,7 @@ The path to the file containing the template to install.
 
 ### templates
 
-Specify index templates in form of hash. Can contain multiple templates.
+Specify index templates (to be created on startup) in the form of a hash (accepts JSON dict). Can contain multiple templates.
 
 ```
 templates { "template_name_1": "path_to_template_1_file", "template_name_2": "path_to_template_2_file"}
@@ -1561,6 +1561,50 @@ Default value is `false`.
 
 **NOTE:** This parameter requests to install elasticsearch-xpack gem.
 
+### data_stream_template_use_index_patterns_wildcard
+
+Specify whether index patterns should include a wildcard (*) when creating an index template. This is particularly useful to prevent errors in scenarios where index templates are generated automatically, and multiple services with distinct suffixes are in use.
+
+Default value is `true`.
+
+Consider the following JSON error response when index patterns clash due to wildcard usage:
+```json
+{
+  "error": {
+    "root_cause": [
+      {
+        "type": "illegal_argument_exception",
+        "reason": "index template [eks-kube-apiserver] has index patterns [eks-kube-apiserver*] matching patterns from existing templates [eks-kube-apiserver-audit] with patterns (eks-kube-apiserver-audit => [eks-kube-apiserver-audit*]) that have the same priority [0], multiple index templates may not match during index creation, please use a different priority"
+      }
+    ],
+    "type": "illegal_argument_exception",
+    "reason": "index template [eks-kube-apiserver] has index patterns [eks-kube-apiserver*] matching patterns from existing templates [eks-kube-apiserver-audit] with patterns (eks-kube-apiserver-audit => [eks-kube-apiserver-audit*]) that have the same priority [0], multiple index templates may not match during index creation, please use a different priority"
+  },
+  "status": 400
+}
+```
+
+#### Usage Examples
+
+When `data_stream_template_use_index_patterns_wildcard` is set to `true` (default):
+
+```
+data_stream_name: foo
+data_stream_template_use_index_patterns_wildcard: true
+```
+
+In this case, the resulting index patterns will be: `["foo*"]`
+
+When `data_stream_template_use_index_patterns_wildcard` is set to `false`:
+
+```
+data_stream_name: foo
+data_stream_template_use_index_patterns_wildcard: false
+```
+
+The resulting index patterns will be: `["foo"]`
+
+
 ## Troubleshooting
 
 See [Troubleshooting document](README.Troubleshooting.md)

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '5.4.2'
+  s.version       = '5.4.4'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -883,11 +883,11 @@ EOC
           if split_request?(bulk_message, info)
             bulk_message.each do |info, msgs|
               send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info, unpackedMsgArr[info]) unless msgs.empty?
+            ensure
               unpackedMsgArr[info].clear
               msgs.clear
               # Clear bulk_message_count for this info.
               bulk_message_count[info] = 0;
-              next
             end
           end
 
@@ -907,7 +907,7 @@ EOC
 
       bulk_message.each do |info, msgs|
         send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info, unpackedMsgArr[info]) unless msgs.empty?
-
+      ensure
         unpackedMsgArr[info].clear
         msgs.clear
       end

data/lib/fluent/plugin/out_elasticsearch_data_stream.rb

@@ -13,6 +13,7 @@ module Fluent::Plugin
     config_param :data_stream_template_name, :string, :default => nil
     config_param :data_stream_ilm_policy, :string, :default => nil
     config_param :data_stream_ilm_policy_overwrite, :bool, :default => false
+    config_param :data_stream_template_use_index_patterns_wildcard, :bool, :default => true
 
     # Elasticsearch 7.9 or later always support new style of index template.
     config_set_default :use_legacy_template, false
@@ -112,8 +113,9 @@ module Fluent::Plugin
 
     def create_index_template(datastream_name, template_name, ilm_name, host = nil)
       return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host)
+      wildcard = @data_stream_template_use_index_patterns_wildcard ? '*' : ''
       body = {
-        "index_patterns" => ["#{datastream_name}*"],
+        "index_patterns" => ["#{datastream_name}#{wildcard}"],
         "data_stream" => {},
         "template" => {
           "settings" => {
@@ -140,7 +142,7 @@ module Fluent::Plugin
         response = client(host).indices.get_data_stream(params)
         return (not response.is_a?(TRANSPORT_CLASS::Transport::Errors::NotFound))
       rescue TRANSPORT_CLASS::Transport::Errors::NotFound => e
-        log.info "Specified data stream does not exist. Will be created: <#{e}>"
+        log.info "Specified data stream does not exist. Will be created: <#{datastream_name}>"
         return false
       end
     end
@@ -217,6 +219,8 @@ module Fluent::Plugin
       data_stream_name = @data_stream_name
       data_stream_template_name = @data_stream_template_name
       data_stream_ilm_name = @data_stream_ilm_name
+      id_key = @id_key
+      remove_keys = @remove_keys
       host = nil
       if @use_placeholder
         host = if @hosts
@@ -255,7 +259,15 @@ module Fluent::Plugin
           unless record.has_key?("@timestamp")
             record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
           end
-          bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)
+
+          # meta variable to be appended in bulk_message
+          meta = {}
+          meta = {'_id' => record[id_key]} if id_key and record[id_key]
+
+          # Remove any key in remove_keys if defined
+          remove_keys.each { |key| record.delete(key) } if remove_keys
+
+          bulk_message = append_record_to_messages(CREATE_OP, meta, headers, record, bulk_message)
         rescue => e
           router.emit_error_event(tag, time, record, e)
         end

data/test/plugin/test_elasticsearch_index_lifecycle_management.rb

@@ -37,6 +37,10 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
     end
   end
 
+  def elasticsearch_miscellaneous_content_type?
+    Gem::Version.create(Elasticsearch::VERSION) >= Gem::Version.new("9.0.0")
+  end
+
   def ilm_existence_endpoint(policy_id)
     if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("8.0.0")
       "_ilm/policy/#{policy_id}"
@@ -90,10 +94,20 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
   def test_create_ilm_policy
     stub_request(:get, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}").
       to_return(:status => 404, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
-    stub_request(:put, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}").
-      with(:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}",
-         :headers => {'Content-Type'=>'application/json'}).
-      to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
+    if elasticsearch_miscellaneous_content_type?
+      stub_request(:put, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}").
+        with(
+          body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}",
+          headers: {
+            'Content-Type'=>'application/vnd.elasticsearch+json; compatible-with=9',
+          }).
+        to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
+    else
+      stub_request(:put, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}").
+        with(:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}",
+             :headers => {'Content-Type'=>'application/json'}).
+        to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
+    end
     stub_elastic_info
     create_ilm_policy("fluent-policy")
 

data/test/plugin/test_out_elasticsearch.rb

@@ -65,6 +65,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
   end
 
+  def elasticsearch_miscellaneous_content_type?
+    Gem::Version.create(Elasticsearch::VERSION) >= Gem::Version.new("9.0.0")
+  end
+
   def default_type_name
     Fluent::Plugin::ElasticsearchOutput::DEFAULT_TYPE_NAME
   end
@@ -3889,7 +3893,12 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_content_type_header
     stub_request(:head, "http://localhost:9200/").
       to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
-    if Elasticsearch::VERSION >= "6.0.2"
+
+    if elasticsearch_miscellaneous_content_type?
+      elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
+                          with(headers: { "Content-Type" => "application/vnd.elasticsearch+x-ndjson; compatible-with=9" }).
+                          to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
+    elsif Elasticsearch::VERSION >= "6.0.2"
       elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
                           with(headers: { "Content-Type" => "application/x-ndjson" }).
                           to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
@@ -5005,7 +5014,8 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
           driver.feed(time.to_i, sample_record.merge({"pipeline_id" => pipeline_id}))
         end
       }
-      assert_equal("could not push logs to Elasticsearch cluster ({:host=>\"myhost-1\", :port=>9200, :scheme=>\"http\"}): [503] ", exception.message)
+      params = {:host=> "myhost-1", :port=>9200, :scheme=>"http"}
+      assert_equal("could not push logs to Elasticsearch cluster (#{params}): [503] ", exception.message)
     end
   end
 

data/test/plugin/test_out_elasticsearch_data_stream.rb

@@ -19,6 +19,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     @driver = nil
     log = Fluent::Engine.log
     log.out.logs.slice!(0, log.out.logs.length)
+    @bulk_meta = []
     @bulk_records = []
   end
 
@@ -34,6 +35,10 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     '_ilm'.freeze
   end
 
+  def index_template_endpoint
+    '_index_template'.freeze
+  end
+
   def driver(conf='', es_version=elasticsearch_version.to_i, client_version=elasticsearch_version)
     # For request stub to detect compatibility.
     @es_version ||= es_version
@@ -83,6 +88,10 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     {'message' => 'Sample record no timestamp'}
   end
 
+  def sample_record_with_hash
+    {'@timestamp' => SAMPLE_RECORD_TIMESTAMP, 'message' => 'Sample record', '_hash' => 'new_id_value'}
+  end
+
   RESPONSE_ACKNOWLEDGED = {"acknowledged": true}
   DUPLICATED_DATA_STREAM_EXCEPTION = {"error": {}, "status": 400}
   NONEXISTENT_DATA_STREAM_EXCEPTION = {"error": {}, "status": 404}
@@ -124,12 +133,15 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     stub_request(:get, "#{url}/_index_template/#{name}").to_return(:status => [404, TRANSPORT_CLASS::Transport::Errors::NotFound], :headers => {'x-elastic-product' => 'Elasticsearch'})
   end
 
-
   def push_bulk_request(req_body)
     # bulk data must be pair of OP and records
     # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
     # {"@timestamp": ...}
     ops = req_body.split("\n")
+    @bulk_meta += ops.values_at(
+      * ops.each_index.select {|i| i.even? }
+    ).map{ |i| JSON.parse(i) }
+
     @bulk_records += ops.values_at(
       * ops.each_index.select {|i| i.odd? }
     ).map{ |i| JSON.parse(i) }
@@ -486,7 +498,8 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
         '@type' => ELASTIC_DATA_STREAM_TYPE,
         'data_stream_name' => 'foo',
         'data_stream_ilm_name' => "foo_ilm_policy",
-        'data_stream_template_name' => "foo_tpl"
+        'data_stream_template_name' => "foo_tpl",
+        'data_stream_template_use_index_patterns_wildcard' => false
       })
     assert_equal "foo", driver(conf).instance.data_stream_name
   end
@@ -556,6 +569,21 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     assert_equal true, driver(config).instance.compression
   end
 
+  def test_configure_wildcard_usage
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    config = %{
+      data_stream_name                                        foo
+      data_stream_template_name                               foo_tpl
+      data_stream_ilm_name                                    foo_ilm_policy
+      data_stream_template_use_index_patterns_wildcard        false
+    }
+
+    stub_default
+
+    assert_equal false, driver(config).instance.data_stream_template_use_index_patterns_wildcard
+  end
+
   def test_check_compression_strategy
     omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
 
@@ -722,6 +750,72 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     assert_equal "foo_policy", driver(conf).instance.data_stream_ilm_name
   end
 
+  def test_template_index_patterns_with_wildcard
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    stub_existent_ilm?
+
+    stub_nonexistent_data_stream?
+    stub_data_stream
+
+    stub_nonexistent_template?("foo_template")
+    stub_request(:put, "http://localhost:9200/#{index_template_endpoint}/foo_template").with do |req|
+      # bulk data must be pair of OP and records
+      # {"create": {}}\nhttp://localhost:9200/_index_template//foo_template
+      # {"@timestamp": ...}
+      push_bulk_request(req.body)
+    end.to_return({:status => 200, :body => "{}", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
+
+    conf = config_element(
+      'ROOT', '', {
+      '@type' => ELASTIC_DATA_STREAM_TYPE,
+      'data_stream_name' => 'foo',
+      'data_stream_ilm_name' => 'foo_ilm_policy',
+      'data_stream_template_use_index_patterns_wildcard' => false
+    })
+
+    assert_nothing_raised {
+      driver(conf)
+    }
+
+    assert_requested(:put, "http://localhost:9200/#{index_template_endpoint}/foo_template",
+                     body: '{"index_patterns":["foo"],"data_stream":{},"template":{"settings":{"index.lifecycle.name":"foo_ilm_policy"}}}',
+                     times: 1)
+  end
+
+  def test_template_index_patterns_without_wildcard
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    stub_existent_ilm?
+
+    stub_nonexistent_data_stream?
+    stub_data_stream
+
+    stub_nonexistent_template?("foo_template")
+    stub_request(:put, "http://localhost:9200/#{index_template_endpoint}/foo_template").with do |req|
+      # bulk data must be pair of OP and records
+      # {"create": {}}\nhttp://localhost:9200/_index_template//foo_template
+      # {"@timestamp": ...}
+      push_bulk_request(req.body)
+    end.to_return({:status => 200, :body => "{}", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
+
+    conf = config_element(
+      'ROOT', '', {
+      '@type' => ELASTIC_DATA_STREAM_TYPE,
+      'data_stream_name' => 'foo',
+      'data_stream_ilm_name' => 'foo_ilm_policy',
+      'data_stream_template_use_index_patterns_wildcard' => true
+    })
+
+    assert_nothing_raised {
+      driver(conf)
+    }
+
+    assert_requested(:put, "http://localhost:9200/#{index_template_endpoint}/foo_template",
+                     body: '{"index_patterns":["foo*"],"data_stream":{},"template":{"settings":{"index.lifecycle.name":"foo_ilm_policy"}}}',
+                     times: 1)
+  end
+
   def test_placeholder
     omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
 
@@ -1111,4 +1205,73 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     assert_equal 1, @bulk_records.length
     assert(@bulk_records[0].has_key?('@timestamp'))
   end
+
+  def test_id_key_if_value_present_in_record
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    stub_default
+    stub_bulk_feed
+    conf = config_element(
+      'ROOT', '', {
+        '@type' => ELASTIC_DATA_STREAM_TYPE,
+        'data_stream_name' => 'foo',
+        'data_stream_ilm_name' => 'foo_ilm_policy',
+        'data_stream_template_name' => 'foo_tpl',
+        'id_key' => '_hash'
+      })
+    driver(conf).run(default_tag: 'test') do
+      driver.feed(sample_record_with_hash)
+    end
+    assert_equal 1, @bulk_records.length
+    assert_equal 1, @bulk_meta.length
+    assert(@bulk_meta[0].has_key?('create'))
+    assert(@bulk_meta[0]['create'].has_key?('_id'))
+    assert_equal 'new_id_value', @bulk_meta[0]['create']['_id']
+
+  end
+
+  def test_id_key_if_value_not_present_in_record
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    stub_default
+    stub_bulk_feed
+    conf = config_element(
+      'ROOT', '', {
+        '@type' => ELASTIC_DATA_STREAM_TYPE,
+        'data_stream_name' => 'foo',
+        'data_stream_ilm_name' => 'foo_ilm_policy',
+        'data_stream_template_name' => 'foo_tpl',
+        'id_key' => '_hash'
+      })
+    driver(conf).run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+    assert_equal 1, @bulk_records.length
+    assert_equal 1, @bulk_meta.length
+    assert(@bulk_meta[0].has_key?('create'))
+    assert_false(@bulk_meta[0]['create'].has_key?('_id'))
+
+  end
+
+  def test_remove_keys_if_key_present_in_record
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    stub_default
+    stub_bulk_feed
+    conf = config_element(
+      'ROOT', '', {
+        '@type' => ELASTIC_DATA_STREAM_TYPE,
+        'data_stream_name' => 'foo',
+        'data_stream_ilm_name' => 'foo_ilm_policy',
+        'data_stream_template_name' => 'foo_tpl',
+        'remove_keys' => '_hash, message'
+      })
+    driver(conf).run(default_tag: 'test') do
+      driver.feed(sample_record_with_hash)
+    end
+    assert_equal 1, @bulk_records.length
+    assert_false(@bulk_records[0].has_key?('_hash'))
+    assert_false(@bulk_records[0].has_key?('message'))
+
+  end
 end

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -50,6 +50,10 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
   end
 
+  def elasticsearch_miscellaneous_content_type?
+    Gem::Version.create(Elasticsearch::VERSION) >= Gem::Version.new("9.0.0")
+  end
+
   def default_type_name
     Fluent::Plugin::ElasticsearchOutput::DEFAULT_TYPE_NAME
   end
@@ -388,7 +392,11 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   def test_content_type_header
     stub_request(:head, "http://localhost:9200/").
       to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
-    if Elasticsearch::VERSION >= "6.0.2"
+    if elasticsearch_miscellaneous_content_type?
+      elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
+                          with(headers: { "Content-Type" => "application/vnd.elasticsearch+x-ndjson; compatible-with=9" }).
+                          to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
+    elsif Elasticsearch::VERSION >= "6.0.2"
       elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
                           with(headers: { "Content-Type" => "application/x-ndjson"}).
                           to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})

metadata

@@ -1,16 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 5.4.2
+  version: 5.4.4
 platform: ruby
 authors:
 - diogo
 - pitr
 - Hiroshi Hatake
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-12-05 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -236,7 +235,6 @@ licenses:
 - Apache-2.0
 metadata:
   changelog_uri: https://github.com/uken/fluent-plugin-elasticsearch/blob/master/History.md
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -251,8 +249,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.6
-signing_key:
+rubygems_version: 3.6.7
 specification_version: 4
 summary: Elasticsearch output plugin for Fluent event collector
 test_files: