fluent-plugin-elasticsearch 5.3.0 → 5.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a475837f544383a6d7c59f43f31c9d79a7c6154b2b8c851c266a9bbbe393e50a
-  data.tar.gz: 300ad2d07e3009f9d50433ceecee0e410fbd129d75c86d71646e659f58f19df3
+  metadata.gz: 61bd10f31500ee04a5f1cbf8f9a25a22c8cf22feac51ce7728c1adec21f63565
+  data.tar.gz: 73f6a791314e73d4669b26105b3377ad99a90948e4f153086c0188484119f5ee
 SHA512:
-  metadata.gz: d1b12f9ee8582952388aadb13e06ef2538677bb761e9e55f1ea41214442c224b3334252a2b78bc35db6a334c19fa2183fff4a9311bd55bf8edeb44cbab30a1b5
-  data.tar.gz: eff7db364b8c307f01935b18221effc879e65d9ef300caba3cbdfc3a1c0fe7d84f3f4d8741fdb934bed553f58d0475a76b235598145e9f39c56c124766015540
+  metadata.gz: 9f2ddddd3297b4c69a1499f0dcd4e3b690224523afacce1787f2490bee809ea2cc6d81cbf723fb3ce4275ca4a9ea83a1e7da908b33ff86c94fa79799fd6cabb4
+  data.tar.gz: cecb42477e365df347ebdfeded701f2faebf1084c01a76b1b4319920c241af6f785cebff47531181de8d3eeea03d50783df5c44c46202b95ef28bb42e366168d

data/.github/workflows/linux.yml

@@ -16,7 +16,7 @@ jobs:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}

data/.github/workflows/macos.yml

@@ -16,7 +16,7 @@ jobs:
           - macOS-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}

data/.github/workflows/windows.yml

@@ -16,7 +16,7 @@ jobs:
           - windows-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}

data/History.md

@@ -2,6 +2,13 @@
 
 ### [Unreleased]
 
+### 5.4.1
+- Adjust ilm endpoint (#1036)
+
+### 5.4.0
+- Fix support for host/hosts placeholders for ipv6 addresses (#1030)
+- Handle newer es library (#1032)
+
 ### 5.3.0
 
 - Unpin `faraday` from v1, upgrade to v2. (#1012)

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '5.3.0'
+  s.version       = '5.4.1'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_compat.rb

@@ -12,6 +12,7 @@ if Gem::Version.new(Elasticsearch::VERSION) < Gem::Version.new("8.0.0")
   begin
     require 'elasticsearch/xpack'
   rescue LoadError
+    require 'elasticsearch/api' # For elasticsearch-ruby 8 or later
   end
 end
 

data/lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb

@@ -45,19 +45,19 @@ module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
   end
 
   def get_ilm_policy
-    if Gem::Version.new(TRANSPORT_CLASS::VERSION) < Gem::Version.new("8.0.0")
-      client.ilm.get_policy
+    if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("8.0.0")
+      client.ilm.get_lifecycle
     else
-      client.enrich.get_policy
+      client.ilm.get_policy
     end
   end
 
   def ilm_policy_exists?(policy_id)
     begin
-      if Gem::Version.new(TRANSPORT_CLASS::VERSION) < Gem::Version.new("8.0.0")
-        client.ilm.get_policy(policy_id: policy_id)
+      if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("8.0.0")
+        client.ilm.get_lifecycle(policy: policy_id)
       else
-        client.enrich.get_policy(name: policy_id)
+        client.ilm.get_policy(policy_id: policy_id)
       end
       true
     rescue
@@ -67,10 +67,10 @@ module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
 
   def ilm_policy_put(policy_id, policy)
     log.info("Installing ILM policy: #{policy}")
-    if Gem::Version.new(TRANSPORT_CLASS::VERSION) < Gem::Version.new("8.0.0")
-      client.ilm.put_policy(policy_id: policy_id, body: policy)
+    if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("8.0.0")
+      client.ilm.put_lifecycle(policy: policy_id, body: policy)
     else
-      client.enrich.put_policy(name: policy_id, body: policy)
+      client.ilm.put_policy(policy_id: policy_id, body: policy)
     end
   end
 

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -653,6 +653,14 @@ EOC
       end
     end
 
+    def is_ipv6_host(host_str)
+      begin
+        IPAddr.new(host_str).ipv6?
+      rescue IPAddr::InvalidAddressError
+        return false
+      end
+    end
+
     def get_connection_options(con_host=nil)
 
       hosts = if con_host || @hosts
@@ -664,6 +672,21 @@ EOC
               port:   (host_str.split(':')[1] || @port).to_i,
               scheme: @scheme.to_s
             }
+          # Support ipv6 for host/host placeholders
+          elsif is_ipv6_host(host_str)
+            if Resolv::IPv6::Regex.match(host_str)
+              {
+                host: "[#{host_str}]",
+                port: @port.to_i,
+                scheme: @scheme.to_s 
+              }
+            else
+              {
+                host: host_str,
+                port: @port.to_i, 
+                scheme: @scheme.to_s
+              }
+            end
           else
             # New hosts format expects URLs such as http://logs.foo.com,https://john:pass@logs2.foo.com/elastic
             uri = URI(get_escaped_userinfo(host_str))

data/lib/fluent/plugin/out_elasticsearch_data_stream.rb

@@ -102,8 +102,8 @@ module Fluent::Plugin
       retry_operate(@max_retry_putting_template,
                     @fail_on_putting_template_retry_exceed,
                     @catch_transport_exception_on_retry) do
-        if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
-          client(host).enrich.put_policy(params.merge(name: ilm_name))
+        if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("8.0.0")
+          client(host).ilm.put_lifecycle(params.merge(policy: ilm_name))
         else
           client(host).xpack.ilm.put_policy(params.merge(policy_id: ilm_name))
         end
@@ -159,8 +159,8 @@ module Fluent::Plugin
 
     def ilm_policy_exists?(policy_id, host = nil)
       begin
-        if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
-          client(host).enrich.get_policy(name: policy_id)
+        if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("8.0.0")
+          client(host).ilm.get_lifecycle(policy: policy_id)
         else
           client(host).ilm.get_policy(policy_id: policy_id)
         end

data/test/helper.rb

@@ -15,7 +15,6 @@ end
 
 require 'test/unit'
 require 'fluent/test'
-require 'minitest/pride'
 
 require 'webmock/test_unit'
 WebMock.disable_net_connect!

data/test/plugin/test_elasticsearch_fallback_selector.rb

@@ -15,7 +15,7 @@ class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
   def stub_elastic(url="http://localhost:9200/_bulk")
     stub_request(:post, url).with do |req|
       @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
-    end
+    end.to_return({:status => 200, :body => "{}", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
   end
 
   def elasticsearch_version

data/test/plugin/test_elasticsearch_index_lifecycle_management.rb

@@ -38,19 +38,15 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
   end
 
   def ilm_existence_endpoint(policy_id)
-    if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
-      "_enrich/policy/#{policy_id}"
+    if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("8.0.0")
+      "_ilm/policy/#{policy_id}"
     else
       "_ilm/policy/%7B:policy_id=%3E%22#{policy_id}%22%7D"
     end
   end
 
   def ilm_creation_endpoint(policy_id)
-    if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
-      "_enrich/policy/#{policy_id}"
-    else
-      "_ilm/policy/#{policy_id}"
-    end
+    "_ilm/policy/#{policy_id}"
   end
 
   def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version)
@@ -60,7 +56,7 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
 
   def test_xpack_info
     stub_request(:get, "http://localhost:9200/_xpack").
-      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json" })
+      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json", 'x-elastic-product' => 'Elasticsearch'  })
     stub_elastic_info
     expected = {"features"=>{"ilm"=>{"available"=>true, "enabled"=>true}}}
     if xpack_info.is_a?(Elasticsearch::API::Response)
@@ -72,32 +68,32 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
 
   def test_verify_ilm_working
     stub_request(:get, "http://localhost:9200/_xpack").
-      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json" })
+      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json", 'x-elastic-product' => 'Elasticsearch'  })
     stub_elastic_info
     assert_nothing_raised { verify_ilm_working }
   end
 
   def test_ilm_policy_doesnt_exists
     stub_request(:get, "http://localhost:9200/#{ilm_existence_endpoint("fluentd-policy")}").
-      to_return(:status => 404, :body => "", :headers => {})
+      to_return(:status => 404, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
     assert_false(ilm_policy_exists?("fluentd-policy"))
   end
 
   def test_ilm_policy_exists
     stub_request(:get, "http://localhost:9200/#{ilm_existence_endpoint("fluent-policy")}").
-      to_return(:status => 200, :body => "", :headers => {})
+      to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
     assert_true(ilm_policy_exists?("fluent-policy"))
   end
 
   def test_create_ilm_policy
     stub_request(:get, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}").
-      to_return(:status => 404, :body => "", :headers => {})
+      to_return(:status => 404, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
     stub_request(:put, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}").
       with(:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}",
          :headers => {'Content-Type'=>'application/json'}).
-      to_return(:status => 200, :body => "", :headers => {})
+      to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
     create_ilm_policy("fluent-policy")
 

data/test/plugin/test_in_elasticsearch.rb

@@ -334,7 +334,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response.to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
 
     driver(CONFIG)
@@ -350,7 +350,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     stub_request(@http_method, "http://localhost:9200/#{index_name}/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
 
     driver(CONFIG + %[index_name #{index_name}])
@@ -366,7 +366,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     stub_request(@http_method, "http://localhost:9200/#{index_name}/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
 
     driver(CONFIG + %[parse_timestamp])
@@ -385,7 +385,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     stub_request(@http_method, "http://localhost:9200/#{index_name}/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
 
     driver(CONFIG + %[parse_timestamp true
@@ -405,7 +405,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response.to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
 
     driver(CONFIG + %[docinfo true])
@@ -425,11 +425,11 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"],\"slice\":{\"id\":0,\"max\":2}}").
       to_return(status: 200, body: sample_response.to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
     stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"],\"slice\":{\"id\":1,\"max\":2}}").
       to_return(status: 200, body: sample_response.to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
 
     driver(CONFIG + %[num_slices 2])
@@ -446,7 +446,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_scroll_response.to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
     connection = 0
     scroll_request = stub_request(@http_method, "http://localhost:9200/_search/scroll?scroll=1m").
       with(
@@ -457,10 +457,10 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     scroll_request.to_return(lambda do |req|
                                if connection <= 1
                                  {status: 200, body: sample_scroll_response_2.to_s,
-                                  headers: {'Content-Type' => 'application/json'}}
+                                  headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'}}
                                else
                                  {status: 200, body: sample_scroll_response_terminate.to_s,
-                                  headers: {'Content-Type' => 'application/json'}}
+                                  headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'}}
                                end
                              end)
     stub_request(:delete, "http://localhost:9200/_search/scroll/WomkoUKG0QPB679Ulo6TqQgh3pIGRUmrl9qXXGK3EeiQh9rbYNasTkspZQcJ01uz").