fluent-plugin-elasticsearch 5.4.3 → 5.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 542a7706becd1b24f8d3e6ad8957015dba5f721b3a93047ec4bea80313df8200
-  data.tar.gz: 8104645d96578abf0f4357a22c83a7f9cbc548e7bbc35ef5eeb8726deaada6f8
+  metadata.gz: 5808f1d08b2a774e67ea22cbf9b678100a1a630c22f6d27ed4c2c656b396f32c
+  data.tar.gz: 836d9697ba68a2b86875429abd31fd09b8d4533d26b2dab4ede5ebdf64a9aad7
 SHA512:
-  metadata.gz: 93a2a932a766170b197f3ded762c89912f743cec3ddcef3e7719d8ca57b504f6f7b51bc17bac02ef5073fd1fade3366140a7d775351c8b86b1f05693a76be122
-  data.tar.gz: 35b82c172909a1f47dc74507c7ea9df451576b9cb1c8dcba85fdb094b3568b2187aafa1aecbd3367ef2ab0a347a30f90d8b663780e3261bdefc5dcb2b24f0133
+  metadata.gz: 3c349e163f308a421d5bc7f74f170def2a1b3845650680b93479a01fcdcd32b2a89dfcb49903ca6c3eaea041beb9dcead4ce5735189f83aa95f9151c2ce3433a
+  data.tar.gz: 0cb7d133409a0289a00fd3e09e9447ac92a3ab72d74cefe92e191477e3bbba0673a322448d8828bb0adc37fcb6f7c523469ec9a95273a5a86fade6fb0dc9cdc8

data/.github/workflows/linux.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.0', '3.1', '3.2' ]
+        ruby: [ '3.0', '3.1', '3.2', '3.3', '3.4' ]
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/macos.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.0', '3.1', '3.2' ]
+        ruby: [ '3.0', '3.1', '3.2', '3.3', '3.4' ]
         os:
           - macOS-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/.github/workflows/windows.yml

@@ -11,7 +11,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.0', '3.1', '3.2' ]
+        ruby: [ '3.0', '3.1', '3.2', '3.3', '3.4' ]
         os:
           - windows-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/History.md

@@ -2,6 +2,12 @@
 
 ### [Unreleased]
 
+### 5.4.4
+- docs: Clarify README docs for template_name and templates (#1043)
+- `out_elasticsearch_data_stream`: Avoid long worrying log line at info level (#1044)
+- `out_elasticsearch_data_stream`: Support `id_key` and `remove_keys` (#1057)
+- `out_elasticsearch`: Fix memory leaks when exception was raised frequently (#1065)
+
 ### 5.4.3
 - add data_stream_template\_use\_index\_patterns\_wildcard in elasticsearch\_data\_stream (#1040)
 

data/README.md

@@ -527,7 +527,7 @@ into same document with data1 as wanted and duplicated document is avoided.
 
 ### template_name
 
-The name of the template to define. If a template by the name given is already present, it will be left unchanged, unless [template_overwrite](#template_overwrite) is set, in which case the template will be updated.
+The name of the index template to create on fluentd startup. If a template by the name given is already present, it will be left unchanged, unless [template_overwrite](#template_overwrite) is set, in which case the template will be updated.
 
 This parameter along with template_file allow the plugin to behave similarly to Logstash (it installs a template at creation time) so that raw records are available. See [https://github.com/uken/fluent-plugin-elasticsearch/issues/33](https://github.com/uken/fluent-plugin-elasticsearch/issues/33).
 
@@ -541,7 +541,7 @@ The path to the file containing the template to install.
 
 ### templates
 
-Specify index templates in form of hash. Can contain multiple templates.
+Specify index templates (to be created on startup) in the form of a hash (accepts JSON dict). Can contain multiple templates.
 
 ```
 templates { "template_name_1": "path_to_template_1_file", "template_name_2": "path_to_template_2_file"}

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '5.4.3'
+  s.version       = '5.4.4'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -883,11 +883,11 @@ EOC
           if split_request?(bulk_message, info)
             bulk_message.each do |info, msgs|
               send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info, unpackedMsgArr[info]) unless msgs.empty?
+            ensure
               unpackedMsgArr[info].clear
               msgs.clear
               # Clear bulk_message_count for this info.
               bulk_message_count[info] = 0;
-              next
             end
           end
 
@@ -907,7 +907,7 @@ EOC
 
       bulk_message.each do |info, msgs|
         send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info, unpackedMsgArr[info]) unless msgs.empty?
-
+      ensure
         unpackedMsgArr[info].clear
         msgs.clear
       end

data/lib/fluent/plugin/out_elasticsearch_data_stream.rb

@@ -142,7 +142,7 @@ module Fluent::Plugin
         response = client(host).indices.get_data_stream(params)
         return (not response.is_a?(TRANSPORT_CLASS::Transport::Errors::NotFound))
       rescue TRANSPORT_CLASS::Transport::Errors::NotFound => e
-        log.info "Specified data stream does not exist. Will be created: <#{e}>"
+        log.info "Specified data stream does not exist. Will be created: <#{datastream_name}>"
         return false
       end
     end
@@ -219,6 +219,8 @@ module Fluent::Plugin
       data_stream_name = @data_stream_name
       data_stream_template_name = @data_stream_template_name
       data_stream_ilm_name = @data_stream_ilm_name
+      id_key = @id_key
+      remove_keys = @remove_keys
       host = nil
       if @use_placeholder
         host = if @hosts
@@ -257,7 +259,15 @@ module Fluent::Plugin
           unless record.has_key?("@timestamp")
             record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
           end
-          bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)
+
+          # meta variable to be appended in bulk_message
+          meta = {}
+          meta = {'_id' => record[id_key]} if id_key and record[id_key]
+
+          # Remove any key in remove_keys if defined
+          remove_keys.each { |key| record.delete(key) } if remove_keys
+
+          bulk_message = append_record_to_messages(CREATE_OP, meta, headers, record, bulk_message)
         rescue => e
           router.emit_error_event(tag, time, record, e)
         end

data/test/plugin/test_elasticsearch_index_lifecycle_management.rb

@@ -37,6 +37,10 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
     end
   end
 
+  def elasticsearch_miscellaneous_content_type?
+    Gem::Version.create(Elasticsearch::VERSION) >= Gem::Version.new("9.0.0")
+  end
+
   def ilm_existence_endpoint(policy_id)
     if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("8.0.0")
       "_ilm/policy/#{policy_id}"
@@ -90,10 +94,20 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
   def test_create_ilm_policy
     stub_request(:get, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}").
       to_return(:status => 404, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
-    stub_request(:put, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}").
-      with(:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}",
-         :headers => {'Content-Type'=>'application/json'}).
-      to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
+    if elasticsearch_miscellaneous_content_type?
+      stub_request(:put, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}").
+        with(
+          body: "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}",
+          headers: {
+            'Content-Type'=>'application/vnd.elasticsearch+json; compatible-with=9',
+          }).
+        to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
+    else
+      stub_request(:put, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}").
+        with(:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}",
+             :headers => {'Content-Type'=>'application/json'}).
+        to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
+    end
     stub_elastic_info
     create_ilm_policy("fluent-policy")
 

data/test/plugin/test_out_elasticsearch.rb

@@ -65,6 +65,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
   end
 
+  def elasticsearch_miscellaneous_content_type?
+    Gem::Version.create(Elasticsearch::VERSION) >= Gem::Version.new("9.0.0")
+  end
+
   def default_type_name
     Fluent::Plugin::ElasticsearchOutput::DEFAULT_TYPE_NAME
   end
@@ -3889,7 +3893,12 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
   def test_content_type_header
     stub_request(:head, "http://localhost:9200/").
       to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
-    if Elasticsearch::VERSION >= "6.0.2"
+
+    if elasticsearch_miscellaneous_content_type?
+      elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
+                          with(headers: { "Content-Type" => "application/vnd.elasticsearch+x-ndjson; compatible-with=9" }).
+                          to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
+    elsif Elasticsearch::VERSION >= "6.0.2"
       elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
                           with(headers: { "Content-Type" => "application/x-ndjson" }).
                           to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
@@ -5005,7 +5014,8 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
           driver.feed(time.to_i, sample_record.merge({"pipeline_id" => pipeline_id}))
         end
       }
-      assert_equal("could not push logs to Elasticsearch cluster ({:host=>\"myhost-1\", :port=>9200, :scheme=>\"http\"}): [503] ", exception.message)
+      params = {:host=> "myhost-1", :port=>9200, :scheme=>"http"}
+      assert_equal("could not push logs to Elasticsearch cluster (#{params}): [503] ", exception.message)
     end
   end
 

data/test/plugin/test_out_elasticsearch_data_stream.rb

@@ -19,6 +19,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     @driver = nil
     log = Fluent::Engine.log
     log.out.logs.slice!(0, log.out.logs.length)
+    @bulk_meta = []
     @bulk_records = []
   end
 
@@ -87,6 +88,10 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     {'message' => 'Sample record no timestamp'}
   end
 
+  def sample_record_with_hash
+    {'@timestamp' => SAMPLE_RECORD_TIMESTAMP, 'message' => 'Sample record', '_hash' => 'new_id_value'}
+  end
+
   RESPONSE_ACKNOWLEDGED = {"acknowledged": true}
   DUPLICATED_DATA_STREAM_EXCEPTION = {"error": {}, "status": 400}
   NONEXISTENT_DATA_STREAM_EXCEPTION = {"error": {}, "status": 404}
@@ -133,6 +138,10 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
     # {"@timestamp": ...}
     ops = req_body.split("\n")
+    @bulk_meta += ops.values_at(
+      * ops.each_index.select {|i| i.even? }
+    ).map{ |i| JSON.parse(i) }
+
     @bulk_records += ops.values_at(
       * ops.each_index.select {|i| i.odd? }
     ).map{ |i| JSON.parse(i) }
@@ -1196,4 +1205,73 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     assert_equal 1, @bulk_records.length
     assert(@bulk_records[0].has_key?('@timestamp'))
   end
+
+  def test_id_key_if_value_present_in_record
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    stub_default
+    stub_bulk_feed
+    conf = config_element(
+      'ROOT', '', {
+        '@type' => ELASTIC_DATA_STREAM_TYPE,
+        'data_stream_name' => 'foo',
+        'data_stream_ilm_name' => 'foo_ilm_policy',
+        'data_stream_template_name' => 'foo_tpl',
+        'id_key' => '_hash'
+      })
+    driver(conf).run(default_tag: 'test') do
+      driver.feed(sample_record_with_hash)
+    end
+    assert_equal 1, @bulk_records.length
+    assert_equal 1, @bulk_meta.length
+    assert(@bulk_meta[0].has_key?('create'))
+    assert(@bulk_meta[0]['create'].has_key?('_id'))
+    assert_equal 'new_id_value', @bulk_meta[0]['create']['_id']
+
+  end
+
+  def test_id_key_if_value_not_present_in_record
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    stub_default
+    stub_bulk_feed
+    conf = config_element(
+      'ROOT', '', {
+        '@type' => ELASTIC_DATA_STREAM_TYPE,
+        'data_stream_name' => 'foo',
+        'data_stream_ilm_name' => 'foo_ilm_policy',
+        'data_stream_template_name' => 'foo_tpl',
+        'id_key' => '_hash'
+      })
+    driver(conf).run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+    assert_equal 1, @bulk_records.length
+    assert_equal 1, @bulk_meta.length
+    assert(@bulk_meta[0].has_key?('create'))
+    assert_false(@bulk_meta[0]['create'].has_key?('_id'))
+
+  end
+
+  def test_remove_keys_if_key_present_in_record
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    stub_default
+    stub_bulk_feed
+    conf = config_element(
+      'ROOT', '', {
+        '@type' => ELASTIC_DATA_STREAM_TYPE,
+        'data_stream_name' => 'foo',
+        'data_stream_ilm_name' => 'foo_ilm_policy',
+        'data_stream_template_name' => 'foo_tpl',
+        'remove_keys' => '_hash, message'
+      })
+    driver(conf).run(default_tag: 'test') do
+      driver.feed(sample_record_with_hash)
+    end
+    assert_equal 1, @bulk_records.length
+    assert_false(@bulk_records[0].has_key?('_hash'))
+    assert_false(@bulk_records[0].has_key?('message'))
+
+  end
 end

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -50,6 +50,10 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
   end
 
+  def elasticsearch_miscellaneous_content_type?
+    Gem::Version.create(Elasticsearch::VERSION) >= Gem::Version.new("9.0.0")
+  end
+
   def default_type_name
     Fluent::Plugin::ElasticsearchOutput::DEFAULT_TYPE_NAME
   end
@@ -388,7 +392,11 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   def test_content_type_header
     stub_request(:head, "http://localhost:9200/").
       to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
-    if Elasticsearch::VERSION >= "6.0.2"
+    if elasticsearch_miscellaneous_content_type?
+      elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
+                          with(headers: { "Content-Type" => "application/vnd.elasticsearch+x-ndjson; compatible-with=9" }).
+                          to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
+    elsif Elasticsearch::VERSION >= "6.0.2"
       elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
                           with(headers: { "Content-Type" => "application/x-ndjson"}).
                           to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})

metadata

@@ -1,16 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 5.4.3
+  version: 5.4.4
 platform: ruby
 authors:
 - diogo
 - pitr
 - Hiroshi Hatake
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-01-15 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -236,7 +235,6 @@ licenses:
 - Apache-2.0
 metadata:
   changelog_uri: https://github.com/uken/fluent-plugin-elasticsearch/blob/master/History.md
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -251,8 +249,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.6
-signing_key:
+rubygems_version: 3.6.7
 specification_version: 4
 summary: Elasticsearch output plugin for Fluent event collector
 test_files: