fluent-plugin-elasticsearch 4.1.1 → 5.4.3

data/test/plugin/test_elasticsearch_index_lifecycle_management.rb

@@ -6,10 +6,15 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
   include Fluent::Plugin::ElasticsearchIndexLifecycleManagement
 
   def setup
-    begin
-      require "elasticsearch/xpack"
-    rescue LoadError
-      omit "ILM testcase needs elasticsearch-xpack gem."
+    if Gem::Version.new(Elasticsearch::VERSION) < Gem::Version.new("7.14.0")
+      begin
+        require "elasticsearch/xpack"
+      rescue LoadError
+        omit "ILM testcase needs elasticsearch-xpack gem."
+      end
+    end
+    if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.4.0")
+      omit "elastisearch-ruby v7.4.0 or later is needed for ILM."
     end
     Fluent::Plugin::ElasticsearchIndexLifecycleManagement.module_eval(<<-CODE)
       def client
@@ -24,40 +29,74 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
     CODE
   end
 
+  def elasticsearch_version
+    if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
+      TRANSPORT_CLASS::VERSION
+    else
+      '6.4.2'.freeze
+    end
+  end
+
+  def ilm_existence_endpoint(policy_id)
+    if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("8.0.0")
+      "_ilm/policy/#{policy_id}"
+    else
+      "_ilm/policy/%7B:policy_id=%3E%22#{policy_id}%22%7D"
+    end
+  end
+
+  def ilm_creation_endpoint(policy_id)
+    "_ilm/policy/#{policy_id}"
+  end
+
+  def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version)
+    body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
+    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
+  end
+
   def test_xpack_info
     stub_request(:get, "http://localhost:9200/_xpack").
-      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json", 'x-elastic-product' => 'Elasticsearch'  })
+    stub_elastic_info
     expected = {"features"=>{"ilm"=>{"available"=>true, "enabled"=>true}}}
-    assert_equal(expected, xpack_info)
+    if xpack_info.is_a?(Elasticsearch::API::Response)
+      assert_equal(expected, xpack_info.body)
+    else
+      assert_equal(expected, xpack_info)
+    end
   end
 
   def test_verify_ilm_working
     stub_request(:get, "http://localhost:9200/_xpack").
-      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
+      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json", 'x-elastic-product' => 'Elasticsearch'  })
+    stub_elastic_info
     assert_nothing_raised { verify_ilm_working }
   end
 
   def test_ilm_policy_doesnt_exists
-    stub_request(:get, "http://localhost:9200/_ilm/policy/%7B:policy_id=%3E%22fluentd-policy%22%7D").
-      to_return(:status => 404, :body => "", :headers => {})
-    assert_false(ilm_policy_exists?(policy_id: "fluentd-policy"))
+    stub_request(:get, "http://localhost:9200/#{ilm_existence_endpoint("fluentd-policy")}").
+      to_return(:status => 404, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
+    stub_elastic_info
+    assert_false(ilm_policy_exists?("fluentd-policy"))
   end
 
   def test_ilm_policy_exists
-    stub_request(:get, "http://localhost:9200/_ilm/policy/%7B:policy_id=%3E%22fluent-policy%22%7D").
-      to_return(:status => 200, :body => "", :headers => {})
-    assert_true(ilm_policy_exists?(policy_id: "fluent-policy"))
+    stub_request(:get, "http://localhost:9200/#{ilm_existence_endpoint("fluent-policy")}").
+      to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
+    stub_elastic_info
+    assert_true(ilm_policy_exists?("fluent-policy"))
   end
 
   def test_create_ilm_policy
-    stub_request(:get, "http://localhost:9200/_ilm/policy/fluent-policy").
-      to_return(:status => 404, :body => "", :headers => {})
-    stub_request(:put, "http://localhost:9200/_ilm/policy/fluent-policy").
+    stub_request(:get, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}").
+      to_return(:status => 404, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
+    stub_request(:put, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}").
       with(:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}",
          :headers => {'Content-Type'=>'application/json'}).
-      to_return(:status => 200, :body => "", :headers => {})
+      to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
+    stub_elastic_info
     create_ilm_policy("fluent-policy")
 
-    assert_requested(:put, "http://localhost:9200/_ilm/policy/fluent-policy", times: 1)
+    assert_requested(:put, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}", times: 1)
   end
 end

data/test/plugin/test_elasticsearch_tls.rb

@@ -62,8 +62,14 @@ class TestElasticsearchTLS < Test::Unit::TestCase
       d = driver('')
       ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version, nil, nil)
       if @use_tls_minmax_version
-        assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
-                      min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
+        if @enabled_tlsv1_3
+          assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
+                        min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
+        else
+          assert_equal({max_version: nil,
+                        min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
+
+        end
       else
         assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION}, ssl_version_options)
       end

data/test/plugin/test_filter_elasticsearch_genid.rb

@@ -132,10 +132,10 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
   end
 
   class UseEntireRecordAsSeedTest < self
-    data("md5" => ["md5", "MuMU0gHOP1cWvvg/J4aEFg=="],
-         "sha1" => ["sha1", "GZ6Iup9Ywyk5spCWtPQbtZnfK0U="],
-         "sha256" => ["sha256", "O4YN0RiXCUAYeaR97UUULRLxgra/R2dvTV47viir5l4="],
-         "sha512" => ["sha512", "FtbwO1xsLUq0KcO0mj0l80rbwFH5rGE3vL+Vgh90+4R/9j+/Ni/ipwhiOoUcetDxj1r5Vf/92B54La+QTu3eMA=="],)
+    data("md5" => ["md5", "OAod7J0DR9s9/rOQnkeSFw=="],
+         "sha1" => ["sha1", "0CT4aMJ4gxMT3TKaYPCYApiVsq8="],
+         "sha256" => ["sha256", "mbAuKF5can0TTj/JBk71AXtOyoVqw5W5gMPUxx6pxLk="],
+         "sha512" => ["sha512", "f7kz5KVuDy+riENePDzqBjGQfbuRNpRBSQMzT2/6hrljXbYtBy3YFmxB86ofIf3zz4ZBao2QM2W7YvcwbRtK1w=="],)
     def test_record
       hash_type, expected = data
       d = create_driver(%[
@@ -151,10 +151,10 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
                    d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
     end
 
-    data("md5" => ["md5", "GJfpWe8ofiGzn97bc9Gh0Q=="],
-         "sha1" => ["sha1", "AVaK67Tz0bEJ8xNEzjOQ6r9fAu4="],
-         "sha256" => ["sha256", "WIXWAuf/Z94Uw95mudloo2bgjhSsSduQIwkKTQsNFgU="],
-         "sha512" => ["sha512", "yjMGGxy8uc7gCrPgm8W6MzJGLFk0GtUwJ6w/91laf6WNywuvG/7T6kNHLagAV8rSW8xzxmtEfyValBO5scuoKw=="],)
+    data("md5" => ["md5", "Hb0jwxofNQP+ufQTKK1U4g=="],
+         "sha1" => ["sha1", "BakTtlotl/u+yOON6YcViTz6nms="],
+         "sha256" => ["sha256", "eLuTCsFqDlk6PfABNyD39r36+yNIBeDTHyNKfJ8fZQw="],
+         "sha512" => ["sha512", "PhPCNGalM4H4xT19DnCBnpwr56lbvCo8wJGyCiH9dWcyhn1nA5l1diYSZlF2fNiq1+wzMqfGvJILIjgQrlAPcg=="],)
     def test_record_with_tag
       hash_type, expected = data
       d = create_driver(%[
@@ -171,10 +171,10 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
                    d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
     end
 
-    data("md5" => ["md5", "5nQSaJ4F1p9rDFign13Lfg=="],
-         "sha1" => ["sha1", "hyo9+0ZFBpizKl2NShs3C8yQcGw="],
-         "sha256" => ["sha256", "romVsZSIksbqYsOSnUzolZQw76ankcy0DgvDZ3CayTo="],
-         "sha512" => ["sha512", "RPU7K2Pt0iVyvV7p5usqcUIIOmfTajD1aa7pkR9qZ89UARH/lpm6ESY9iwuYJj92lxOUuF5OxlEwvV7uXJ07iA=="],)
+    data("md5" => ["md5", "C8vfhC4kecNCNutFCuC6MA=="],
+         "sha1" => ["sha1", "+YWVqUEL90wpKJRrionUJwNgXHg="],
+         "sha256" => ["sha256", "eSqGZqjnO6Uum/4CNfJaolX49+2XKogiGMHGNHiO91Q="],
+         "sha512" => ["sha512", "iVmuD0D+i/WtBwNza09ZXSIW8Xg8/yrUwK/M/EZaCMjz/x5FyyCiVkb1VVKsgNnJy0SYt4w21dhHewu1aXM6HA=="],)
     def test_record_with_time
       hash_type, expected = data
       d = create_driver(%[
@@ -191,10 +191,10 @@ class ElasticsearchGenidFilterTest < Test::Unit::TestCase
                    d.filtered.map {|e| e.last}.first[d.instance.hash_id_key])
     end
 
-    data("md5" => ["md5", "zGQF35KlMUibJAcgkgQDtw=="],
-         "sha1" => ["sha1", "1x9RZO1xEuWps090qq4DUIsU9x8="],
-         "sha256" => ["sha256", "eulMz0eF56lBEf31aIs0OG2TGCH/aoPfZbRqfEOkAwk="],
-         "sha512" => ["sha512", "mIiYATtpdUFEFCIZg1FdKssIs7oWY0gJjhSSbet0ddUmqB+CiQAcAMTmrXO6AVSH0vsMvao/8vtC8AsIPfF1fA=="],)
+    data("md5" => ["md5", "lU7d4EiF+2M1zxWcsmBbjg=="],
+         "sha1" => ["sha1", "nghmz1y3KTEFxalfS2/Oe4n4yfQ="],
+         "sha256" => ["sha256", "d0le9UOnUeuGPF/2yEBRM1YzOYeHtxYOE1UU6JgJrvU="],
+         "sha512" => ["sha512", "n7rhisGHUBne6c4Cs9DRMbPror8O5Y/vYajDqAtOaiUTys/Z1EKBMnZQA0iVNFw7joX33cenBW3Yyccct3xSew=="],)
     def test_record_with_tag_and_time
       hash_type, expected = data
       d = create_driver(%[

data/test/plugin/test_in_elasticsearch.rb

@@ -20,12 +20,30 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     @driver = nil
     log = Fluent::Engine.log
     log.out.logs.slice!(0, log.out.logs.length)
+    @http_method = if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("7.9.0")
+                     :post
+                   else
+                     :get
+                   end
   end
 
   def driver(conf='')
     @driver ||= Fluent::Test::Driver::Input.new(Fluent::Plugin::ElasticsearchInput).configure(conf)
   end
 
+  def elasticsearch_version
+    if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
+      TRANSPORT_CLASS::VERSION
+    else
+      '7.9.0'.freeze
+    end
+  end
+
+  def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version)
+    body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
+    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
+  end
+
   def sample_response(index_name="fluentd")
     {
       "took"=>4,
@@ -313,10 +331,11 @@ class ElasticsearchInputTest < Test::Unit::TestCase
   end
 
   def test_emit
-    stub_request(:get, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
+    stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response.to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
+    stub_elastic_info
 
     driver(CONFIG)
     driver.run(expect_emits: 1, timeout: 10)
@@ -328,10 +347,11 @@ class ElasticsearchInputTest < Test::Unit::TestCase
 
   def test_emit_with_custom_index_name
     index_name = "logstash"
-    stub_request(:get, "http://localhost:9200/#{index_name}/_search?scroll=1m&size=1000").
+    stub_request(@http_method, "http://localhost:9200/#{index_name}/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
+    stub_elastic_info
 
     driver(CONFIG + %[index_name #{index_name}])
     driver.run(expect_emits: 1, timeout: 10)
@@ -343,10 +363,11 @@ class ElasticsearchInputTest < Test::Unit::TestCase
 
   def test_emit_with_parse_timestamp
     index_name = "fluentd"
-    stub_request(:get, "http://localhost:9200/#{index_name}/_search?scroll=1m&size=1000").
+    stub_request(@http_method, "http://localhost:9200/#{index_name}/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
+    stub_elastic_info
 
     driver(CONFIG + %[parse_timestamp])
     driver.run(expect_emits: 1, timeout: 10)
@@ -361,10 +382,11 @@ class ElasticsearchInputTest < Test::Unit::TestCase
 
   def test_emit_with_parse_timestamp_and_timstamp_format
     index_name = "fluentd"
-    stub_request(:get, "http://localhost:9200/#{index_name}/_search?scroll=1m&size=1000").
+    stub_request(@http_method, "http://localhost:9200/#{index_name}/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
+    stub_elastic_info
 
     driver(CONFIG + %[parse_timestamp true
                       timestamp_key_format %Y-%m-%dT%H:%M:%S.%N%z
@@ -380,10 +402,11 @@ class ElasticsearchInputTest < Test::Unit::TestCase
   end
 
   def test_emit_with_docinfo
-    stub_request(:get, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
+    stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response.to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
+    stub_elastic_info
 
     driver(CONFIG + %[docinfo true])
     driver.run(expect_emits: 1, timeout: 10)
@@ -399,14 +422,15 @@ class ElasticsearchInputTest < Test::Unit::TestCase
   end
 
   def test_emit_with_slices
-    stub_request(:get, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
+    stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"],\"slice\":{\"id\":0,\"max\":2}}").
       to_return(status: 200, body: sample_response.to_s,
-                headers: {'Content-Type' => 'application/json'})
-    stub_request(:get, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
+    stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"],\"slice\":{\"id\":1,\"max\":2}}").
       to_return(status: 200, body: sample_response.to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
+    stub_elastic_info
 
     driver(CONFIG + %[num_slices 2])
     driver.run(expect_emits: 1, timeout: 10)
@@ -419,28 +443,34 @@ class ElasticsearchInputTest < Test::Unit::TestCase
   end
 
   def test_emit_with_size
-    stub_request(:get, "http://localhost:9200/fluentd/_search?scroll=1m&size=1").
+    stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_scroll_response.to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
     connection = 0
-    scroll_request = stub_request(:get, "http://localhost:9200/_search/scroll?scroll=1m").
+    scroll_request = stub_request(@http_method, "http://localhost:9200/_search/scroll?scroll=1m").
       with(
         body: "{\"scroll_id\":\"WomkoUKG0QPB679Ulo6TqQgh3pIGRUmrl9qXXGK3EeiQh9rbYNasTkspZQcJ01uz\"}") do
       connection += 1
     end
+    stub_elastic_info
     scroll_request.to_return(lambda do |req|
                                if connection <= 1
                                  {status: 200, body: sample_scroll_response_2.to_s,
-                                  headers: {'Content-Type' => 'application/json'}}
+                                  headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'}}
                                else
                                  {status: 200, body: sample_scroll_response_terminate.to_s,
-                                  headers: {'Content-Type' => 'application/json'}}
+                                  headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'}}
                                end
                              end)
-    stub_request(:delete, "http://localhost:9200/_search/scroll/WomkoUKG0QPB679Ulo6TqQgh3pIGRUmrl9qXXGK3EeiQh9rbYNasTkspZQcJ01uz").
+    if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("7.0.0")
+      stub_request(:delete, "http://localhost:9200/_search/scroll").
+        with(body: "{\"scroll_id\":\"WomkoUKG0QPB679Ulo6TqQgh3pIGRUmrl9qXXGK3EeiQh9rbYNasTkspZQcJ01uz\"}").
       to_return(status: 200, body: "", headers: {})
-
+    else
+      stub_request(:delete, "http://localhost:9200/_search/scroll/WomkoUKG0QPB679Ulo6TqQgh3pIGRUmrl9qXXGK3EeiQh9rbYNasTkspZQcJ01uz").
+        to_return(status: 200, body: "", headers: {})
+    end
     driver(CONFIG + %[size 1])
     driver.run(expect_emits: 1, timeout: 10)
     expected = [

data/test/plugin/test_index_alias_template.json

@@ -0,0 +1,11 @@
+{
+  "priority": 105,
+  "index_patterns": "--index_prefix-----appid---*",
+  "template" : {
+    "settings": {},
+    "mappings": {},
+    "aliases": {
+      "--appid---alias": {}
+    }
+  }
+}

data/test/plugin/test_index_template.json

@@ -0,0 +1,25 @@
+{
+  "index_patterns": "te*",
+  "template": {
+    "settings": {
+      "number_of_shards": 1
+    },
+    "mappings": {
+      "type1": {
+        "_source": {
+          "enabled": false
+        },
+        "properties": {
+          "host_name": {
+            "type": "string",
+            "index": "not_analyzed"
+          },
+          "created_at": {
+            "type": "date",
+            "format": "EEE MMM dd HH:mm:ss Z YYYY"
+          }
+        }
+      }
+    }
+  }
+}