fluent-plugin-elasticsearch 5.3.0 → 5.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a475837f544383a6d7c59f43f31c9d79a7c6154b2b8c851c266a9bbbe393e50a
-  data.tar.gz: 300ad2d07e3009f9d50433ceecee0e410fbd129d75c86d71646e659f58f19df3
+  metadata.gz: 4fc25cc0c5f9b35ce8ab9d733cbd4d908df838540f2bc5aad0fce57267f2c6d5
+  data.tar.gz: 1179e1275edf74186b41c3ff004236a940df58b4c7214f2fe6e3a7ae3852cd81
 SHA512:
-  metadata.gz: d1b12f9ee8582952388aadb13e06ef2538677bb761e9e55f1ea41214442c224b3334252a2b78bc35db6a334c19fa2183fff4a9311bd55bf8edeb44cbab30a1b5
-  data.tar.gz: eff7db364b8c307f01935b18221effc879e65d9ef300caba3cbdfc3a1c0fe7d84f3f4d8741fdb934bed553f58d0475a76b235598145e9f39c56c124766015540
+  metadata.gz: 43f42f1611c24eb8ef00c153bf8ed4acc07da4a02535f5df6eecc44d9a2d219a15de1551131e7efeb80c8b8e2c20ef203302d95a2c1fe369d3996920654ec381
+  data.tar.gz: 10de0f5a8e5c2179595a99b01a331cbc102c1d9529a59435d8f5c079236cf4132f86d05e552ae2e08e1bce88ec8eac8cd28c721c016a5cb17e13526cf5336d9e

data/.github/workflows/linux.yml

@@ -16,7 +16,7 @@ jobs:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}

data/.github/workflows/macos.yml

@@ -16,7 +16,7 @@ jobs:
           - macOS-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}

data/.github/workflows/windows.yml

@@ -16,7 +16,7 @@ jobs:
           - windows-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}

data/History.md

@@ -2,6 +2,10 @@
 
 ### [Unreleased]
 
+### 5.4.0
+- Fix support for host/hosts placeholders for ipv6 addresses (#1030)
+- Handle newer es library (#1032)
+
 ### 5.3.0
 
 - Unpin `faraday` from v1, upgrade to v2. (#1012)

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '5.3.0'
+  s.version       = '5.4.0'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -653,6 +653,14 @@ EOC
       end
     end
 
+    def is_ipv6_host(host_str)
+      begin
+        IPAddr.new(host_str).ipv6?
+      rescue IPAddr::InvalidAddressError
+        return false
+      end
+    end
+
     def get_connection_options(con_host=nil)
 
       hosts = if con_host || @hosts
@@ -664,6 +672,21 @@ EOC
               port:   (host_str.split(':')[1] || @port).to_i,
               scheme: @scheme.to_s
             }
+          # Support ipv6 for host/host placeholders
+          elsif is_ipv6_host(host_str)
+            if Resolv::IPv6::Regex.match(host_str)
+              {
+                host: "[#{host_str}]",
+                port: @port.to_i,
+                scheme: @scheme.to_s 
+              }
+            else
+              {
+                host: host_str,
+                port: @port.to_i, 
+                scheme: @scheme.to_s
+              }
+            end
           else
             # New hosts format expects URLs such as http://logs.foo.com,https://john:pass@logs2.foo.com/elastic
             uri = URI(get_escaped_userinfo(host_str))

data/test/helper.rb

@@ -15,7 +15,6 @@ end
 
 require 'test/unit'
 require 'fluent/test'
-require 'minitest/pride'
 
 require 'webmock/test_unit'
 WebMock.disable_net_connect!

data/test/plugin/test_elasticsearch_fallback_selector.rb

@@ -15,7 +15,7 @@ class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
   def stub_elastic(url="http://localhost:9200/_bulk")
     stub_request(:post, url).with do |req|
       @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
-    end
+    end.to_return({:status => 200, :body => "{}", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
   end
 
   def elasticsearch_version

data/test/plugin/test_elasticsearch_index_lifecycle_management.rb

@@ -60,7 +60,7 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
 
   def test_xpack_info
     stub_request(:get, "http://localhost:9200/_xpack").
-      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json" })
+      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json", 'x-elastic-product' => 'Elasticsearch'  })
     stub_elastic_info
     expected = {"features"=>{"ilm"=>{"available"=>true, "enabled"=>true}}}
     if xpack_info.is_a?(Elasticsearch::API::Response)
@@ -72,32 +72,32 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
 
   def test_verify_ilm_working
     stub_request(:get, "http://localhost:9200/_xpack").
-      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json" })
+      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json", 'x-elastic-product' => 'Elasticsearch'  })
     stub_elastic_info
     assert_nothing_raised { verify_ilm_working }
   end
 
   def test_ilm_policy_doesnt_exists
     stub_request(:get, "http://localhost:9200/#{ilm_existence_endpoint("fluentd-policy")}").
-      to_return(:status => 404, :body => "", :headers => {})
+      to_return(:status => 404, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
     assert_false(ilm_policy_exists?("fluentd-policy"))
   end
 
   def test_ilm_policy_exists
     stub_request(:get, "http://localhost:9200/#{ilm_existence_endpoint("fluent-policy")}").
-      to_return(:status => 200, :body => "", :headers => {})
+      to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
     assert_true(ilm_policy_exists?("fluent-policy"))
   end
 
   def test_create_ilm_policy
     stub_request(:get, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}").
-      to_return(:status => 404, :body => "", :headers => {})
+      to_return(:status => 404, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
     stub_request(:put, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}").
       with(:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}",
          :headers => {'Content-Type'=>'application/json'}).
-      to_return(:status => 200, :body => "", :headers => {})
+      to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
     create_ilm_policy("fluent-policy")
 

data/test/plugin/test_in_elasticsearch.rb

@@ -334,7 +334,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response.to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
 
     driver(CONFIG)
@@ -350,7 +350,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     stub_request(@http_method, "http://localhost:9200/#{index_name}/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
 
     driver(CONFIG + %[index_name #{index_name}])
@@ -366,7 +366,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     stub_request(@http_method, "http://localhost:9200/#{index_name}/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
 
     driver(CONFIG + %[parse_timestamp])
@@ -385,7 +385,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     stub_request(@http_method, "http://localhost:9200/#{index_name}/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
 
     driver(CONFIG + %[parse_timestamp true
@@ -405,7 +405,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response.to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
 
     driver(CONFIG + %[docinfo true])
@@ -425,11 +425,11 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"],\"slice\":{\"id\":0,\"max\":2}}").
       to_return(status: 200, body: sample_response.to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
     stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"],\"slice\":{\"id\":1,\"max\":2}}").
       to_return(status: 200, body: sample_response.to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
 
     driver(CONFIG + %[num_slices 2])
@@ -446,7 +446,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_scroll_response.to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
     connection = 0
     scroll_request = stub_request(@http_method, "http://localhost:9200/_search/scroll?scroll=1m").
       with(
@@ -457,10 +457,10 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     scroll_request.to_return(lambda do |req|
                                if connection <= 1
                                  {status: 200, body: sample_scroll_response_2.to_s,
-                                  headers: {'Content-Type' => 'application/json'}}
+                                  headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'}}
                                else
                                  {status: 200, body: sample_scroll_response_terminate.to_s,
-                                  headers: {'Content-Type' => 'application/json'}}
+                                  headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'}}
                                end
                              end)
     stub_request(:delete, "http://localhost:9200/_search/scroll/WomkoUKG0QPB679Ulo6TqQgh3pIGRUmrl9qXXGK3EeiQh9rbYNasTkspZQcJ01uz").