fluent-plugin-elasticsearch 5.2.0 → 5.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4c4d872817c5c5f788b40c89adfddec72dc5765666ab90046a38d5e881ba4f2d
-  data.tar.gz: 8e83854fa3eea144ba97e44ba52885dd5f28ef3801fe293c69cf5158e7839789
+  metadata.gz: 425652fd5cb31671a87378fdaaf8fa911aa6bda0743fe44c53d7631d5018aebb
+  data.tar.gz: d3fb4697a32f8448fdd9995adeef50a96839eb15b10f984a03618044c7a9ff35
 SHA512:
-  metadata.gz: 65eb51a5cd710eb39c54f822f3748bb6ba202ebc87d9aab563adf4befec9696279219f452cc99a1d7528e4c484e03807f195cba8695ec16cc0b3525793a872be
-  data.tar.gz: 3ce726708a2b1900f27999518a56bbf857c28337ce1e3d82ab0acd8fbdb58d92488ddb7e9ccf5a8482768b2c4da1c261b96835f02237321b1a941c6d3b0eb2dc
+  metadata.gz: d8b6b4e919d79de0153707bfb84033ea403c0a88f7d76a7f1d986a45e67a0fec340762bd0a5c8776e2f6dff18b267550623fe516018a634fa3c56b9f63a3937c
+  data.tar.gz: 40b798377a202b4e7c0c0e6f7e84910cf7574711a83a0919c274ff5b8b9eb0dd5b90bd3e5c682ba2be9d684384c29bbbfe67d62d9307e8d0df050ee92d460c0b

data/History.md

@@ -2,6 +2,11 @@
 
 ### [Unreleased]
 
+### 5.2.1
+- respect include\_tag\_key and tag\_key setting when using data streams (#936)
+- Handle unsupported version error (#956)
+- Display deprecated warning on ES dynamic plugin (#955)
+
 ### 5.2.0
 -  Migrate to handle Elasticsearch 8 (#949)
 

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '5.2.0'
+  s.version       = '5.2.1'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_compat.rb

@@ -25,3 +25,6 @@ begin
   ::SELECTOR_CLASS = Elasticsearch::Transport::Transport::Connections::Selector
 rescue LoadError
 end
+unless defined?(Elasticsearch::UnsupportedProductError)
+  class Elasticsearch::UnsupportedProductError < StandardError; end
+end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -492,7 +492,11 @@ EOC
     end
 
     def detect_es_major_version
-      @_es_info ||= client.info
+      begin
+        @_es_info ||= client.info
+      rescue Elasticsearch::UnsupportedProductError => e
+        raise Fluent::ConfigError, "Using Elasticsearch client #{client_library_version} is not compatible for your Elasticsearch server. Please check your using elasticsearch gem version and Elasticsearch server."
+      end
       begin
         unless version = @_es_info.dig("version", "number")
           version = @default_elasticsearch_version

data/lib/fluent/plugin/out_elasticsearch_data_stream.rb

@@ -243,6 +243,10 @@ module Fluent::Plugin
       chunk.msgpack_each do |time, record|
         next unless record.is_a? Hash
 
+        if @include_tag_key
+          record[@tag_key] = tag
+        end
+
         begin
           record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
           bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -28,6 +28,8 @@ module Fluent::Plugin
         @dynamic_config[key] = value.to_s
       }
       # end eval all configs
+
+      log.warn "Elasticsearch dynamic plugin will be deprecated and removed in the future. Please consider to use normal Elasticsearch plugin"
     end
 
     def create_meta_config_map

data/test/plugin/test_out_elasticsearch.rb

@@ -920,6 +920,67 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     end
   end
 
+  class GetElasticsearchIncompatibleVersionTest < self
+    def create_driver(conf='', client_version="7.14")
+      # For request stub to detect compatibility.
+      @client_version ||= client_version
+      # Ensure original implementation existence.
+      Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
+        def detect_es_major_version
+          begin
+            @_es_info ||= client.info
+          rescue Elasticsearch::UnsupportedProductError => e
+            raise Fluent::ConfigError, "Using Elasticsearch client #{@client_version} is not compatible for your Elasticsearch server. Please check your using elasticsearch gem version and Elasticsearch server."
+          end
+          begin
+            unless version = @_es_info.dig("version", "number")
+              version = @default_elasticsearch_version
+            end
+          rescue NoMethodError => e
+            log.warn "#{@_es_info} can not dig version information. Assuming Elasticsearch #{@default_elasticsearch_version}", error: e
+            version = @default_elasticsearch_version
+          end
+          version.to_i
+        end
+      CODE
+      Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
+        def client_library_version
+          #{@client_version}
+        end
+      CODE
+      Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput).configure(conf)
+    end
+
+    def test_incompatible_es_version
+      if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.14.0")
+        omit "This test is not effective before elasticsearch 7.14"
+      end
+      config = %{
+        host            logs.google.com
+        port            778
+        scheme          https
+        path            /es/
+        user            john
+        password        doe
+        verify_es_version_at_startup true
+        max_retry_get_es_version 1
+      }
+
+      connection_resets = 0
+      stub_request(:get, "https://logs.google.com:778/es//").
+        with(basic_auth: ['john', 'doe']) do |req|
+        connection_resets += 1
+        raise Elasticsearch::UnsupportedProductError
+      end
+
+      assert_raise(Fluent::ConfigError) do
+        create_driver(config)
+      end
+
+      assert_equal(1, connection_resets)
+    end
+  end
+
   class GetElasticsearchVersionWithFallbackTest < self
     def create_driver(conf='', client_version="\"5.0\"")
       # For request stub to detect compatibility.

data/test/plugin/test_out_elasticsearch_data_stream.rb

@@ -19,7 +19,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     @driver = nil
     log = Fluent::Engine.log
     log.out.logs.slice!(0, log.out.logs.length)
-    @bulk_records = 0
+    @bulk_records = []
   end
 
   def elasticsearch_version
@@ -115,6 +115,17 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     stub_request(:get, "#{url}/_index_template/#{name}").to_return(:status => [404, TRANSPORT_CLASS::Transport::Errors::NotFound])
   end
 
+
+  def push_bulk_request(req_body)
+    # bulk data must be pair of OP and records
+    # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
+    # {"@timestamp": ...}
+    ops = req_body.split("\n")
+    @bulk_records += ops.values_at(
+      * ops.each_index.select {|i| i.odd? }
+    ).map{ |i| JSON.parse(i) }
+  end
+
   def stub_nonexistent_template_retry?(name="foo_tpl", url="http://localhost:9200")
     stub_request(:get, "#{url}/_index_template/#{name}").
       to_return({ status: 500, body: 'Internal Server Error' }, { status: 404, body: '{}' })
@@ -125,19 +136,19 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
       # bulk data must be pair of OP and records
       # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
       # {"@timestamp": ...}
-      @bulk_records += req.body.split("\n").size / 2
+      push_bulk_request(req.body)
     end
     stub_request(:post, "#{url}/#{ilm_name}/_bulk").with do |req|
       # bulk data must be pair of OP and records
       # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
       # {"@timestamp": ...}
-      @bulk_records += req.body.split("\n").size / 2
+      push_bulk_request(req.body)
     end
     stub_request(:post, "#{url}/#{template_name}/_bulk").with do |req|
       # bulk data must be pair of OP and records
       # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
       # {"@timestamp": ...}
-      @bulk_records += req.body.split("\n").size / 2
+      push_bulk_request(req.body)
     end
   end
 
@@ -600,7 +611,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     driver(conf).run(default_tag: 'test') do
       driver.feed(sample_record)
     end
-    assert_equal 1, @bulk_records
+    assert_equal 1, @bulk_records.length
   end
 
   def test_placeholder_params_unset
@@ -619,7 +630,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     driver(conf).run(default_tag: 'test') do
       driver.feed(sample_record)
     end
-    assert_equal 1, @bulk_records
+    assert_equal 1, @bulk_records.length
   end
 
 
@@ -645,7 +656,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     driver(conf).run(default_tag: 'test') do
       driver.feed(sample_record)
     end
-    assert_equal 1, @bulk_records
+    assert_equal 1, @bulk_records.length
   end
 
   def test_custom_record_placeholder
@@ -675,7 +686,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
         driver.feed(record)
       end
     end
-    assert_equal keys.count, @bulk_records
+    assert_equal keys.count, @bulk_records.length
   end
 
   def test_bulk_insert_feed
@@ -693,7 +704,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     driver(conf).run(default_tag: 'test') do
       driver.feed(sample_record)
     end
-    assert_equal 1, @bulk_records
+    assert_equal 1, @bulk_records.length
   end
 
   def test_template_retry_install_fails
@@ -813,4 +824,70 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
       body: '{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}',
       times: 1)
   end
+
+  def test_doesnt_add_tag_key_when_not_configured
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    config = %{
+      data_stream_name           foo
+      data_stream_template_name  foo_tpl
+      data_stream_ilm_name       foo_ilm_policy
+    }
+
+    stub_default
+    stub_bulk_feed
+    driver(config)
+    driver.run(default_tag: 'mytag') do
+      driver.feed(sample_record)
+    end
+
+    assert_equal(1, @bulk_records.length)
+    assert_false(@bulk_records[0].has_key?('tag'))
+  end
+
+
+  def test_adds_tag_key_when_configured
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    config = %{
+      data_stream_name           foo
+      data_stream_template_name  foo_tpl
+      data_stream_ilm_name       foo_ilm_policy
+      include_tag_key            true
+    }
+
+    stub_default
+    stub_bulk_feed
+    driver(config)
+    driver.run(default_tag: 'mytag') do 
+      driver.feed(sample_record)
+    end
+
+    assert_equal(1, @bulk_records.length)
+    assert(@bulk_records[0].has_key?('tag'))
+    assert_equal('mytag', @bulk_records[0]['tag'])
+  end
+
+  def test_adds_custom_tag_key_when_configured
+    omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
+
+    config = %{
+      data_stream_name           foo
+      data_stream_template_name  foo_tpl
+      data_stream_ilm_name       foo_ilm_policy
+      include_tag_key            true
+      tag_key                    custom_tag_key
+    }
+
+    stub_default
+    stub_bulk_feed
+    driver(config)
+    driver.run(default_tag: 'mytag') do 
+      driver.feed(sample_record)
+    end
+
+    assert_equal(1, @bulk_records.length)
+    assert(@bulk_records[0].has_key?('custom_tag_key'))
+    assert_equal('mytag', @bulk_records[0]['custom_tag_key'])
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 5.2.0
+  version: 5.2.1
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-21 00:00:00.000000000 Z
+date: 2022-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd