fluent-plugin-elasticsearch 5.1.5 → 5.2.0

data/test/plugin/test_out_elasticsearch_data_stream.rb

@@ -22,7 +22,23 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     @bulk_records = 0
   end
 
-  def driver(conf='', es_version=5, client_version="\"5.0\"")
+  def elasticsearch_version
+    if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
+      TRANSPORT_CLASS::VERSION
+    else
+      '5.0.0'.freeze
+    end
+  end
+
+  def ilm_endpoint
+    if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
+      '_enrich'.freeze
+    else
+      '_ilm'.freeze
+    end
+  end
+
+  def driver(conf='', es_version=elasticsearch_version.to_i, client_version=elasticsearch_version)
     # For request stub to detect compatibility.
     @es_version ||= es_version
     @client_version ||= client_version
@@ -63,7 +79,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
   NONEXISTENT_DATA_STREAM_EXCEPTION = {"error": {}, "status": 404}
 
   def stub_ilm_policy(name="foo_ilm_policy", url="http://localhost:9200")
-    stub_request(:put, "#{url}/_ilm/policy/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
+    stub_request(:put, "#{url}/#{ilm_endpoint}/policy/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
   end
 
   def stub_index_template(name="foo_tpl", url="http://localhost:9200")
@@ -79,7 +95,8 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
   end
 
   def stub_existent_ilm?(name="foo_ilm_policy", url="http://localhost:9200")
-    stub_request(:get, "#{url}/_ilm/policy/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
+
+    stub_request(:get, "#{url}/#{ilm_endpoint}/policy/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
   end
 
   def stub_existent_template?(name="foo_tpl", url="http://localhost:9200")
@@ -87,15 +104,15 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
   end
 
   def stub_nonexistent_data_stream?(name="foo", url="http://localhost:9200")
-    stub_request(:get, "#{url}/_data_stream/#{name}").to_return(:status => [404, Elasticsearch::Transport::Transport::Errors::NotFound])
+    stub_request(:get, "#{url}/_data_stream/#{name}").to_return(:status => [404, TRANSPORT_CLASS::Transport::Errors::NotFound])
   end
 
   def stub_nonexistent_ilm?(name="foo_ilm_policy", url="http://localhost:9200")
-    stub_request(:get, "#{url}/_ilm/policy/#{name}").to_return(:status => [404, Elasticsearch::Transport::Transport::Errors::NotFound])
+    stub_request(:get, "#{url}/#{ilm_endpoint}/policy/#{name}").to_return(:status => [404, TRANSPORT_CLASS::Transport::Errors::NotFound])
   end
 
   def stub_nonexistent_template?(name="foo_tpl", url="http://localhost:9200")
-    stub_request(:get, "#{url}/_index_template/#{name}").to_return(:status => [404, Elasticsearch::Transport::Transport::Errors::NotFound])
+    stub_request(:get, "#{url}/_index_template/#{name}").to_return(:status => [404, TRANSPORT_CLASS::Transport::Errors::NotFound])
   end
 
   def stub_nonexistent_template_retry?(name="foo_tpl", url="http://localhost:9200")
@@ -124,9 +141,9 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     end
   end
 
-  def stub_elastic_info(url="http://localhost:9200/", version="7.9.0", headers={})
+  def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version, headers={})
     body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
-    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' }.merge(headers) })
+    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' }.merge(headers) })
   end
 
   def stub_default(datastream_name="foo", ilm_name="foo_ilm_policy", template_name="foo_tpl", host="http://localhost:9200")
@@ -140,7 +157,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
   end
 
   def data_stream_supported?
-    Gem::Version.create(::Elasticsearch::Transport::VERSION) >= Gem::Version.create("7.9.0")
+    Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.create("7.9.0")
   end
 
   # ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
@@ -468,9 +485,9 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
       password         default_password
       data_stream_name default
     }
-    stub_elastic_info("https://host1:443/elastic//", "7.9.0",
+    stub_elastic_info("https://host1:443/elastic//", elasticsearch_version,
                          {'Authorization'=>"Basic #{Base64.encode64('john:password').split.first}"})
-    stub_elastic_info("http://host2/default_path/_data_stream/default", "7.9.0",
+    stub_elastic_info("http://host2/default_path/_data_stream/default", elasticsearch_version,
                          {'Authorization'=>"Basic #{Base64.encode64('john:password').split.first}"})
     stub_existent_data_stream?("default", "https://host1/elastic/")
     instance = driver(config).instance
@@ -730,13 +747,13 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     stub_existent_ilm?
     stub_data_stream
 
-    stub_request(:put, "http://localhost:9200/_ilm/policy/foo_ilm_policy").
+    stub_request(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy").
       to_return(:status => 200, :body => "", :headers => {})
 
     assert_nothing_raised {
       driver(config)
     }
-    assert_requested(:put, "http://localhost:9200/_ilm/policy/foo_ilm_policy", times: 0)
+    assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy", times: 0)
   end
 
   def test_updates_ilm_policy_if_overwrite_set
@@ -755,15 +772,15 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     stub_existent_ilm?
     stub_data_stream
 
-    stub_request(:put, "http://localhost:9200/_ilm/policy/foo_ilm_policy").
+    stub_request(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy").
       to_return(:status => 200, :body => "", :headers => {})
 
     assert_nothing_raised {
       driver(config)
     }
 
-    assert_requested(:put, "http://localhost:9200/_ilm/policy/foo_ilm_policy", times: 1)
-    assert_requested(:put, "http://localhost:9200/_ilm/policy/foo_ilm_policy",
+    assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy", times: 1)
+    assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy",
       body: '{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}',
       times: 1)
   end
@@ -784,15 +801,15 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
     stub_nonexistent_ilm?
     stub_nonexistent_template?("foo_template")
 
-    stub_request(:put, "http://localhost:9200/_ilm/policy/foo_ilm_policy").
+    stub_request(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy").
       to_return(:status => 200, :body => "", :headers => {})
 
     assert_nothing_raised {
       driver(config)
     }
 
-    assert_requested(:put, "http://localhost:9200/_ilm/policy/foo_ilm_policy", times: 1)
-    assert_requested(:put, "http://localhost:9200/_ilm/policy/foo_ilm_policy",
+    assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy", times: 1)
+    assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy",
       body: '{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}',
       times: 1)
   end

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -16,7 +16,15 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     @driver = nil
   end
 
-  def driver(conf='', es_version=5)
+  def elasticsearch_version
+    if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
+      TRANSPORT_CLASS::VERSION
+    else
+      '6.4.2'.freeze
+    end
+  end
+
+  def driver(conf='', es_version=elasticsearch_version.to_i)
     # For request stub to detect compatibility.
     @es_version ||= es_version
     Fluent::Plugin::ElasticsearchOutputDynamic.module_eval(<<-CODE)
@@ -35,7 +43,11 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   end
 
   def elasticsearch_transport_layer_decoupling?
-    Gem::Version.create(::Elasticsearch::Transport::VERSION) >= Gem::Version.new("7.14.0")
+    Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
+  end
+
+  def elastic_transport_layer?
+    Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
   end
 
   def default_type_name
@@ -58,9 +70,9 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     end
   end
 
-  def stub_elastic_info(url="http://localhost:9200/", version="6.4.2")
+  def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version)
     body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
-    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
+    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
   end
 
   def stub_elastic_unavailable(url="http://localhost:9200/_bulk")
@@ -132,7 +144,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   end
 
   test 'configure compression' do
-    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
+    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
 
     config = %{
       compression_level best_compression
@@ -143,7 +155,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   end
 
   test 'check compression strategy' do
-    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
+    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
 
     config = %{
       compression_level best_speed
@@ -154,14 +166,16 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   end
 
   test 'check content-encoding header with compression' do
-    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
+    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
 
     config = %{
       compression_level best_compression
     }
     instance = driver(config).instance
 
-    if elasticsearch_transport_layer_decoupling?
+    if elastic_transport_layer?
+      assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
+    elsif elasticsearch_transport_layer_decoupling?
       assert_equal nil, instance.client.transport.transport.options[:transport_options][:headers]["Content-Encoding"]
     else
       assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
@@ -175,7 +189,9 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     end
     compressable = instance.compressable_connection
 
-    if elasticsearch_transport_layer_decoupling?
+    if elastic_transport_layer?
+      assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
+    elsif elasticsearch_transport_layer_decoupling?
       assert_equal "gzip", instance.client(nil, compressable).transport.transport.options[:transport_options][:headers]["Content-Encoding"]
     else
       assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
@@ -183,14 +199,16 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   end
 
   test 'check compression option is passed to transport' do
-    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
+    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
 
     config = %{
       compression_level best_compression
     }
     instance = driver(config).instance
 
-    if elasticsearch_transport_layer_decoupling?
+    if elastic_transport_layer?
+      assert_equal false, instance.client.transport.options[:compression]
+    elsif elasticsearch_transport_layer_decoupling?
       assert_equal false, instance.client.transport.transport.options[:compression]
     else
       assert_equal false, instance.client.transport.options[:compression]
@@ -204,7 +222,9 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     end
     compressable = instance.compressable_connection
 
-    if elasticsearch_transport_layer_decoupling?
+    if elastic_transport_layer?
+      assert_equal true, instance.client(nil, compressable).transport.options[:compression]
+    elsif elasticsearch_transport_layer_decoupling?
       assert_equal true, instance.client(nil, compressable).transport.transport.options[:compression]
     else
       assert_equal true, instance.client(nil, compressable).transport.options[:compression]
@@ -402,7 +422,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   end
 
   def test_writes_to_default_index_with_compression
-    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.2.0")
+    omit "elastisearch-ruby v7.2.0 or later is needed." if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.2.0")
 
     config = %[
       compression_level default_compression
@@ -441,7 +461,13 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
-    assert_equal(default_type_name, index_cmds.first['index']['_type'])
+    if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
+      assert_nil(index_cmds.first['index']['_type'])
+    elsif Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.0.0")
+      assert_equal("_doc", index_cmds.first['index']['_type'])
+    else
+      assert_equal("fluentd", index_cmds.first['index']['_type'])
+    end
   end
 
   def test_writes_to_specified_index
@@ -471,7 +497,13 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
-    assert_equal('mytype', index_cmds.first['index']['_type'])
+    if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
+      assert_nil(index_cmds.first['index']['_type'])
+    elsif Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.0.0")
+      assert_equal("_doc", index_cmds.first['index']['_type'])
+    else
+      assert_equal("mytype", index_cmds.first['index']['_type'])
+    end
   end
 
   def test_writes_to_specified_host
@@ -971,7 +1003,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
       end
-      assert_equal('routing', index_cmds[0]['index']['_routing'])
+      routing_key = driver.instance.instance_variable_get(:@routing_key_name)
+      assert_equal('routing', index_cmds[0]['index'][routing_key])
     end
 
     def test_adds_nested_routing_key_with_dollar_dot
@@ -981,7 +1014,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
       end
-      assert_equal('routing', index_cmds[0]['index']['_routing'])
+      routing_key = driver.instance.instance_variable_get(:@routing_key_name)
+      assert_equal('routing', index_cmds[0]['index'][routing_key])
     end
 
     def test_adds_nested_routing_key_with_bracket
@@ -991,7 +1025,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
       driver.run(default_tag: 'test') do
         driver.feed(nested_sample_record)
       end
-      assert_equal('routing', index_cmds[0]['index']['_routing'])
+      routing_key = driver.instance.instance_variable_get(:@routing_key_name)
+      assert_equal('routing', index_cmds[0]['index'][routing_key])
     end
   end
 
@@ -1002,7 +1037,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
-    assert(!index_cmds[0]['index'].has_key?('_routing'))
+    routing_key = driver.instance.instance_variable_get(:@routing_key_name)
+    assert(!index_cmds[0]['index'].has_key?(routing_key))
   end
 
   def test_adds_routing_key_when_not_configured
@@ -1011,7 +1047,8 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     driver.run(default_tag: 'test') do
       driver.feed(sample_record)
     end
-    assert(!index_cmds[0]['index'].has_key?('_routing'))
+    routing_key = driver.instance.instance_variable_get(:@routing_key_name)
+    assert(!index_cmds[0]['index'].has_key?(routing_key))
   end
 
   def test_remove_one_key

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 5.1.5
+  version: 5.2.0
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-02-02 00:00:00.000000000 Z
+date: 2022-02-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -171,6 +171,7 @@ files:
 - gemfiles/Gemfile.elasticsearch.v6
 - lib/fluent/log-ext.rb
 - lib/fluent/plugin/default-ilm-policy.json
+- lib/fluent/plugin/elasticsearch_compat.rb
 - lib/fluent/plugin/elasticsearch_constants.rb
 - lib/fluent/plugin/elasticsearch_error.rb
 - lib/fluent/plugin/elasticsearch_error_handler.rb