fluent-plugin-elasticsearch 5.4.3 → 6.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 542a7706becd1b24f8d3e6ad8957015dba5f721b3a93047ec4bea80313df8200
-  data.tar.gz: 8104645d96578abf0f4357a22c83a7f9cbc548e7bbc35ef5eeb8726deaada6f8
+  metadata.gz: b54470881e62c1a0c6fccc56867faf07ecbe210a0926a8073fc646fcd2ffc9e1
+  data.tar.gz: f973db55d17915664f6cb59676738c62f7408c3d94734cdb54b76025d62d82c8
 SHA512:
-  metadata.gz: 93a2a932a766170b197f3ded762c89912f743cec3ddcef3e7719d8ca57b504f6f7b51bc17bac02ef5073fd1fade3366140a7d775351c8b86b1f05693a76be122
-  data.tar.gz: 35b82c172909a1f47dc74507c7ea9df451576b9cb1c8dcba85fdb094b3568b2187aafa1aecbd3367ef2ab0a347a30f90d8b663780e3261bdefc5dcb2b24f0133
+  metadata.gz: 45a60caf34b6e02e5e2a46fd02535a3fa1648d1feb712c1d20c1a88f041c29a39fe688afba1c88511a572075db7e98c767ad81b4197b6bc56a1e1e27d9149764
+  data.tar.gz: 2e0d0dab94d7c3112b637f185c9df0bf16c2fb1805d9186de2fecf6d25e66eea269482431b2db20b62e66f16ebcbd66dd576d6e1e957c12c29d68728b7132806

data/.github/workflows/linux.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.0', '3.1', '3.2' ]
+        ruby: [ '3.0', '3.1', '3.2', '3.3', '3.4' ]
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/macos.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.0', '3.1', '3.2' ]
+        ruby: [ '3.0', '3.1', '3.2', '3.3', '3.4' ]
         os:
           - macOS-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/windows.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.0', '3.1', '3.2' ]
+        ruby: [ '3.0', '3.1', '3.2', '3.3', '3.4' ]
         os:
           - windows-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/History.md

@@ -2,6 +2,15 @@
 
 ### [Unreleased]
 
+### 6.0.0
+-  Adjust Content-Type and Accept headers for sending logs into ES9 (#1064)
+
+### 5.4.4
+- docs: Clarify README docs for template_name and templates (#1043)
+- `out_elasticsearch_data_stream`: Avoid long worrying log line at info level (#1044)
+- `out_elasticsearch_data_stream`: Support `id_key` and `remove_keys` (#1057)
+- `out_elasticsearch`: Fix memory leaks when exception was raised frequently (#1065)
+
 ### 5.4.3
 - add data_stream_template\_use\_index\_patterns\_wildcard in elasticsearch\_data\_stream (#1040)
 

data/README.md

@@ -527,7 +527,7 @@ into same document with data1 as wanted and duplicated document is avoided.
 
 ### template_name
 
-The name of the template to define. If a template by the name given is already present, it will be left unchanged, unless [template_overwrite](#template_overwrite) is set, in which case the template will be updated.
+The name of the index template to create on fluentd startup. If a template by the name given is already present, it will be left unchanged, unless [template_overwrite](#template_overwrite) is set, in which case the template will be updated.
 
 This parameter along with template_file allow the plugin to behave similarly to Logstash (it installs a template at creation time) so that raw records are available. See [https://github.com/uken/fluent-plugin-elasticsearch/issues/33](https://github.com/uken/fluent-plugin-elasticsearch/issues/33).
 
@@ -541,7 +541,7 @@ The path to the file containing the template to install.
 
 ### templates
 
-Specify index templates in form of hash. Can contain multiple templates.
+Specify index templates (to be created on startup) in the form of a hash (accepts JSON dict). Can contain multiple templates.
 
 ```
 templates { "template_name_1": "path_to_template_1_file", "template_name_2": "path_to_template_2_file"}

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '5.4.3'
+  s.version       = '6.0.0'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -73,6 +73,7 @@ module Fluent::Plugin
     DEFAULT_RELOAD_AFTER = -1
     DEFAULT_TARGET_BULK_BYTES = -1
     DEFAULT_POLICY_ID = "logstash-policy"
+    ES9_CONTENT_TYPE = "application/vnd.elasticsearch+x-ndjson; compatible-with=9"
 
     config_param :host, :string,  :default => 'localhost'
     config_param :port, :integer, :default => 9200
@@ -350,10 +351,22 @@ EOC
           log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
           @type_name = '_doc'.freeze
         end
-        if @last_seen_major_version >= 8 && @type_name != DEFAULT_TYPE_NAME_ES_7x
-          log.debug "Detected ES 8.x or above: This parameter has no effect."
+        if @last_seen_major_version == 8 && @type_name != DEFAULT_TYPE_NAME_ES_7x
+          log.debug "Detected ES 8.x: This parameter has no effect."
           @type_name = nil
         end
+        if @last_seen_major_version >= 9
+          if @type_name != DEFAULT_TYPE_NAME_ES_7x
+            log.debug "Detected ES 9.x or above: This parameter has no effect."
+            @type_name = nil
+          end
+          @accept_type = nil
+          if @content_type != ES9_CONTENT_TYPE
+            log.trace "Detected ES 9.x or above: Content-Type will be adjusted."
+            @content_type = ES9_CONTENT_TYPE
+            @accept_type = ES9_CONTENT_TYPE
+          end
+        end
       end
 
       if @validate_client_version && !dry_run?
@@ -614,6 +627,8 @@ EOC
                     .merge(@custom_headers)
                     .merge(@api_key_header)
                     .merge(gzip_headers)
+        headers.merge!('Accept' => @accept_type) if @accept_type
+
         ssl_options = { verify: @ssl_verify, ca_file: @ca_file}.merge(@ssl_version_options)
 
         transport = TRANSPORT_CLASS::Transport::HTTP::Faraday.new(connection_options.merge(
@@ -883,11 +898,11 @@ EOC
           if split_request?(bulk_message, info)
             bulk_message.each do |info, msgs|
               send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info, unpackedMsgArr[info]) unless msgs.empty?
+            ensure
               unpackedMsgArr[info].clear
               msgs.clear
               # Clear bulk_message_count for this info.
               bulk_message_count[info] = 0;
-              next
             end
           end
 
@@ -907,7 +922,7 @@ EOC
 
       bulk_message.each do |info, msgs|
         send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info, unpackedMsgArr[info]) unless msgs.empty?
-
+      ensure
         unpackedMsgArr[info].clear
         msgs.clear
       end

data/lib/fluent/plugin/out_elasticsearch_data_stream.rb

@@ -142,7 +142,7 @@ module Fluent::Plugin
         response = client(host).indices.get_data_stream(params)
         return (not response.is_a?(TRANSPORT_CLASS::Transport::Errors::NotFound))
       rescue TRANSPORT_CLASS::Transport::Errors::NotFound => e
-        log.info "Specified data stream does not exist. Will be created: <#{e}>"
+        log.info "Specified data stream does not exist. Will be created: <#{datastream_name}>"
         return false
       end
     end
@@ -219,6 +219,8 @@ module Fluent::Plugin
       data_stream_name = @data_stream_name
       data_stream_template_name = @data_stream_template_name
       data_stream_ilm_name = @data_stream_ilm_name
+      id_key = @id_key
+      remove_keys = @remove_keys
       host = nil
       if @use_placeholder
         host = if @hosts
@@ -257,7 +259,15 @@ module Fluent::Plugin
           unless record.has_key?("@timestamp")
             record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
           end
-          bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)
+
+          # meta variable to be appended in bulk_message
+          meta = {}
+          meta = {'_id' => record[id_key]} if id_key and record[id_key]
+
+          # Remove any key in remove_keys if defined
+          remove_keys.each { |key| record.delete(key) } if remove_keys
+
+          bulk_message = append_record_to_messages(CREATE_OP, meta, headers, record, bulk_message)
         rescue => e
           router.emit_error_event(tag, time, record, e)
         end

data/test/plugin/test_elasticsearch_index_lifecycle_management.rb

@@ -37,6 +37,10 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
     end
   end
 
+  def elasticsearch_miscellaneous_content_type?
+    Gem::Version.create(Elasticsearch::VERSION) >= Gem::Version.new("9.0.0")
+  end
+
   def ilm_existence_endpoint(policy_id)
     if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("8.0.0")
       "_ilm/policy/#{policy_id}"
@@ -90,10 +94,20 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
   def test_create_ilm_policy
     stub_request(:get, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}").
       to_return(:status => 404, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
-    stub_request(:put, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}").
-      with(:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}",
-         :headers => {'Content-Type'=>'application/json'}).
-      to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
+    if elasticsearch_miscellaneous_content_type?
+      stub_request(:put, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}").
+        with(
+          body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}",
+          headers: {
+            'Content-Type'=>'application/vnd.elasticsearch+json; compatible-with=9',
+          }).
+        to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
+    else
+      stub_request(:put, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}").
+        with(:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}",
+             :headers => {'Content-Type'=>'application/json'}).
+        to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
+    end
     stub_elastic_info
     create_ilm_policy("fluent-policy")
 

data/test/plugin/test_out_elasticsearch.rb

@@ -65,6 +65,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
   end
 
+  def elasticsearch_miscellaneous_content_type?
+    Gem::Version.create(Elasticsearch::VERSION) >= Gem::Version.new("9.0.0")
+  end
+
   def default_type_name
     Fluent::Plugin::ElasticsearchOutput::DEFAULT_TYPE_NAME
   end
@@ -86,7 +90,13 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   end
 
   def stub_elastic(url="http://localhost:9200/_bulk")
-    stub_request(:post, url).with do |req|
+    headers = if elasticsearch_miscellaneous_content_type?
+                { "Content-Type" => "application/vnd.elasticsearch+x-ndjson; compatible-with=9" }
+              else
+                {}
+              end
+
+    stub_request(:post, url).with({headers: headers}) do |req|
       @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
     end.to_return({:status => 200, :body => "", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
   end
@@ -3889,7 +3899,12 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_content_type_header
     stub_request(:head, "http://localhost:9200/").
       to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
-    if Elasticsearch::VERSION >= "6.0.2"
+
+    if elasticsearch_miscellaneous_content_type?
+      elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
+                          with(headers: { "Content-Type" => "application/vnd.elasticsearch+x-ndjson; compatible-with=9" }).
+                          to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
+    elsif Elasticsearch::VERSION >= "6.0.2"
       elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
                           with(headers: { "Content-Type" => "application/x-ndjson" }).
                           to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
@@ -4007,6 +4022,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   data('Elasticsearch 6' => [6, Fluent::Plugin::ElasticsearchOutput::DEFAULT_TYPE_NAME],
        'Elasticsearch 7' => [7, Fluent::Plugin::ElasticsearchOutput::DEFAULT_TYPE_NAME_ES_7x],
        'Elasticsearch 8' => [8, nil],
+       'Elasticsearch 9' => [9, nil],
       )
   def test_writes_to_default_type(data)
     version, index_type = data
@@ -5005,7 +5021,8 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
           driver.feed(time.to_i, sample_record.merge({"pipeline_id" => pipeline_id}))
         end
       }
-      assert_equal("could not push logs to Elasticsearch cluster ({:host=>\"myhost-1\", :port=>9200, :scheme=>\"http\"}): [503] ", exception.message)
+      params = {:host=> "myhost-1", :port=>9200, :scheme=>"http"}
+      assert_equal("could not push logs to Elasticsearch cluster (#{params}): [503] ", exception.message)
     end
   end
 

data/test/plugin/test_out_elasticsearch_data_stream.rb

@@ -19,6 +19,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     @driver = nil
     log = Fluent::Engine.log
     log.out.logs.slice!(0, log.out.logs.length)
+    @bulk_meta = []
     @bulk_records = []
   end
 
@@ -87,6 +88,10 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     {'message' => 'Sample record no timestamp'}
   end
 
+  def sample_record_with_hash
+    {'@timestamp' => SAMPLE_RECORD_TIMESTAMP, 'message' => 'Sample record', '_hash' => 'new_id_value'}
+  end
+
   RESPONSE_ACKNOWLEDGED = {"acknowledged": true}
   DUPLICATED_DATA_STREAM_EXCEPTION = {"error": {}, "status": 400}
   NONEXISTENT_DATA_STREAM_EXCEPTION = {"error": {}, "status": 404}
@@ -133,6 +138,10 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
     # {"@timestamp": ...}
     ops = req_body.split("\n")
+    @bulk_meta += ops.values_at(
+      * ops.each_index.select {|i| i.even? }
+    ).map{ |i| JSON.parse(i) }
+
     @bulk_records += ops.values_at(
       * ops.each_index.select {|i| i.odd? }
     ).map{ |i| JSON.parse(i) }
@@ -1196,4 +1205,73 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     assert_equal 1, @bulk_records.length
     assert(@bulk_records[0].has_key?('@timestamp'))
   end
+
+  def test_id_key_if_value_present_in_record
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    stub_default
+    stub_bulk_feed
+    conf = config_element(
+      'ROOT', '', {
+        '@type' => ELASTIC_DATA_STREAM_TYPE,
+        'data_stream_name' => 'foo',
+        'data_stream_ilm_name' => 'foo_ilm_policy',
+        'data_stream_template_name' => 'foo_tpl',
+        'id_key' => '_hash'
+      })
+    driver(conf).run(default_tag: 'test') do
+      driver.feed(sample_record_with_hash)
+    end
+    assert_equal 1, @bulk_records.length
+    assert_equal 1, @bulk_meta.length
+    assert(@bulk_meta[0].has_key?('create'))
+    assert(@bulk_meta[0]['create'].has_key?('_id'))
+    assert_equal 'new_id_value', @bulk_meta[0]['create']['_id']
+
+  end
+
+  def test_id_key_if_value_not_present_in_record
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    stub_default
+    stub_bulk_feed
+    conf = config_element(
+      'ROOT', '', {
+        '@type' => ELASTIC_DATA_STREAM_TYPE,
+        'data_stream_name' => 'foo',
+        'data_stream_ilm_name' => 'foo_ilm_policy',
+        'data_stream_template_name' => 'foo_tpl',
+        'id_key' => '_hash'
+      })
+    driver(conf).run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+    assert_equal 1, @bulk_records.length
+    assert_equal 1, @bulk_meta.length
+    assert(@bulk_meta[0].has_key?('create'))
+    assert_false(@bulk_meta[0]['create'].has_key?('_id'))
+
+  end
+
+  def test_remove_keys_if_key_present_in_record
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    stub_default
+    stub_bulk_feed
+    conf = config_element(
+      'ROOT', '', {
+        '@type' => ELASTIC_DATA_STREAM_TYPE,
+        'data_stream_name' => 'foo',
+        'data_stream_ilm_name' => 'foo_ilm_policy',
+        'data_stream_template_name' => 'foo_tpl',
+        'remove_keys' => '_hash, message'
+      })
+    driver(conf).run(default_tag: 'test') do
+      driver.feed(sample_record_with_hash)
+    end
+    assert_equal 1, @bulk_records.length
+    assert_false(@bulk_records[0].has_key?('_hash'))
+    assert_false(@bulk_records[0].has_key?('message'))
+
+  end
 end

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -50,6 +50,10 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
   end
 
+  def elasticsearch_miscellaneous_content_type?
+    Gem::Version.create(Elasticsearch::VERSION) >= Gem::Version.new("9.0.0")
+  end
+
   def default_type_name
     Fluent::Plugin::ElasticsearchOutput::DEFAULT_TYPE_NAME
   end
@@ -388,7 +392,11 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   def test_content_type_header
     stub_request(:head, "http://localhost:9200/").
       to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
-    if Elasticsearch::VERSION >= "6.0.2"
+    if elasticsearch_miscellaneous_content_type?
+      elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
+                          with(headers: { "Content-Type" => "application/vnd.elasticsearch+x-ndjson; compatible-with=9" }).
+                          to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
+    elsif Elasticsearch::VERSION >= "6.0.2"
       elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
                           with(headers: { "Content-Type" => "application/x-ndjson"}).
                           to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 5.4.3
+  version: 6.0.0
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-01-15 00:00:00.000000000 Z
+date: 2025-05-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd