fluent-plugin-elasticsearch 5.2.5 → 5.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e6354575b148be2666c68a6a3c72fcda76139517570203751217cb130fcc070
-  data.tar.gz: 32a0c118516bb5dca54e402e655309565d08f2e1d01caf467a39f2c15d58f26f
+  metadata.gz: 4fc25cc0c5f9b35ce8ab9d733cbd4d908df838540f2bc5aad0fce57267f2c6d5
+  data.tar.gz: 1179e1275edf74186b41c3ff004236a940df58b4c7214f2fe6e3a7ae3852cd81
 SHA512:
-  metadata.gz: 444ef6890245dd454adde27b32478369c913f8a4a4b55bd51025390fcf8850ffceb08203fd85265a3fbbdb9f8f4e7bf77526da2119860de1a6c35fe525275ef9
-  data.tar.gz: a78b3e755f9a9f4eef63fb694cc39ceb6db5bc3d565f46f1934b234d00ddb48ea76235db8623f84e70a64afd844f29dc90b75d4d237194054e0777a92f5d199c
+  metadata.gz: 43f42f1611c24eb8ef00c153bf8ed4acc07da4a02535f5df6eecc44d9a2d219a15de1551131e7efeb80c8b8e2c20ef203302d95a2c1fe369d3996920654ec381
+  data.tar.gz: 10de0f5a8e5c2179595a99b01a331cbc102c1d9529a59435d8f5c079236cf4132f86d05e552ae2e08e1bce88ec8eac8cd28c721c016a5cb17e13526cf5336d9e

data/.github/workflows/linux.yml

@@ -11,12 +11,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.7', '3.0', '3.1', '3.2' ]
+        ruby: [ '3.0', '3.1', '3.2' ]
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}

data/.github/workflows/macos.yml

@@ -11,12 +11,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.7', '3.0', '3.1', '3.2' ]
+        ruby: [ '3.0', '3.1', '3.2' ]
         os:
           - macOS-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}

data/.github/workflows/windows.yml

@@ -11,12 +11,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.7', '3.0', '3.1', '3.2' ]
+        ruby: [ '3.0', '3.1', '3.2' ]
         os:
           - windows-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}

data/History.md

@@ -2,6 +2,16 @@
 
 ### [Unreleased]
 
+### 5.4.0
+- Fix support for host/hosts placeholders for ipv6 addresses (#1030)
+- Handle newer es library (#1032)
+
+### 5.3.0
+
+- Unpin `faraday` from v1, upgrade to v2. (#1012)
+  Note that if you can't migrate other plugins from `faraday` v1 yet, need to keep
+  fluent-plugin-elasticsearch v5.2.5.
+
 ### 5.2.5
 
 - Update the name of parameter about `thread_pool`.

data/README.md

@@ -125,6 +125,8 @@ Current maintainers: [Hiroshi Hatake | @cosmo0920](https://github.com/cosmo0920)
 | >= 2.0.0 && < 3.2.3          | >= v0.14.20 | >= 2.1 |
 |  < 2.0.0                     | >= v0.12.0  | >= 1.9 |
 
+NOTE: Since fluent-plugin-elasticsearch 5.3.0, it requires faraday 2.0 or later.
+
 NOTE: For v0.12 version, you should use 1.x.y version. Please send patch into v0.12 branch if you encountered 1.x version's bug.
 
 NOTE: This documentation is for fluent-plugin-elasticsearch 2.x or later. For 1.x documentation, please see [v0.12 branch](https://github.com/uken/fluent-plugin-elasticsearch/tree/v0.12).

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '5.2.5'
+  s.version       = '5.4.0'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}
@@ -23,7 +23,8 @@ Gem::Specification.new do |s|
   s.required_ruby_version = Gem::Requirement.new(">= 2.3".freeze)
 
   s.add_runtime_dependency 'fluentd', '>= 0.14.22'
-  s.add_runtime_dependency "faraday", "~> 1.10"
+  s.add_runtime_dependency 'faraday', '>= 2.0.0'
+  s.add_runtime_dependency 'faraday-excon', '>= 2.0.0'
   s.add_runtime_dependency 'excon', '>= 0'
   s.add_runtime_dependency 'elasticsearch'
 

data/lib/fluent/plugin/in_elasticsearch.rb

@@ -1,5 +1,6 @@
 require 'elasticsearch'
 
+require 'faraday/excon'
 require 'fluent/log-ext'
 require 'fluent/plugin/input'
 require_relative 'elasticsearch_constants'

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -32,6 +32,8 @@ begin
 rescue LoadError
 end
 
+require 'faraday/excon'
+
 module Fluent::Plugin
   class ElasticsearchOutput < Output
     class RecoverableRequestFailure < StandardError; end
@@ -471,7 +473,7 @@ EOC
       when :excon
         { client_key: @client_key, client_cert: @client_cert, client_key_pass: @client_key_pass, nonblock: @http_backend_excon_nonblock }
       when :typhoeus
-        require 'typhoeus'
+        require 'faraday/typhoeus'
         { sslkey: @client_key, sslcert: @client_cert, keypasswd: @client_key_pass }
       end
     rescue LoadError => ex
@@ -651,6 +653,14 @@ EOC
       end
     end
 
+    def is_ipv6_host(host_str)
+      begin
+        IPAddr.new(host_str).ipv6?
+      rescue IPAddr::InvalidAddressError
+        return false
+      end
+    end
+
     def get_connection_options(con_host=nil)
 
       hosts = if con_host || @hosts
@@ -662,6 +672,21 @@ EOC
               port:   (host_str.split(':')[1] || @port).to_i,
               scheme: @scheme.to_s
             }
+          # Support ipv6 for host/host placeholders
+          elsif is_ipv6_host(host_str)
+            if Resolv::IPv6::Regex.match(host_str)
+              {
+                host: "[#{host_str}]",
+                port: @port.to_i,
+                scheme: @scheme.to_s 
+              }
+            else
+              {
+                host: host_str,
+                port: @port.to_i, 
+                scheme: @scheme.to_s
+              }
+            end
           else
             # New hosts format expects URLs such as http://logs.foo.com,https://john:pass@logs2.foo.com/elastic
             uri = URI(get_escaped_userinfo(host_str))

data/test/helper.rb

@@ -15,7 +15,6 @@ end
 
 require 'test/unit'
 require 'fluent/test'
-require 'minitest/pride'
 
 require 'webmock/test_unit'
 WebMock.disable_net_connect!

data/test/plugin/test_elasticsearch_fallback_selector.rb

@@ -15,7 +15,7 @@ class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
   def stub_elastic(url="http://localhost:9200/_bulk")
     stub_request(:post, url).with do |req|
       @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
-    end
+    end.to_return({:status => 200, :body => "{}", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
   end
 
   def elasticsearch_version

data/test/plugin/test_elasticsearch_index_lifecycle_management.rb

@@ -60,7 +60,7 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
 
   def test_xpack_info
     stub_request(:get, "http://localhost:9200/_xpack").
-      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json" })
+      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json", 'x-elastic-product' => 'Elasticsearch'  })
     stub_elastic_info
     expected = {"features"=>{"ilm"=>{"available"=>true, "enabled"=>true}}}
     if xpack_info.is_a?(Elasticsearch::API::Response)
@@ -72,32 +72,32 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
 
   def test_verify_ilm_working
     stub_request(:get, "http://localhost:9200/_xpack").
-      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json" })
+      to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json", 'x-elastic-product' => 'Elasticsearch'  })
     stub_elastic_info
     assert_nothing_raised { verify_ilm_working }
   end
 
   def test_ilm_policy_doesnt_exists
     stub_request(:get, "http://localhost:9200/#{ilm_existence_endpoint("fluentd-policy")}").
-      to_return(:status => 404, :body => "", :headers => {})
+      to_return(:status => 404, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
     assert_false(ilm_policy_exists?("fluentd-policy"))
   end
 
   def test_ilm_policy_exists
     stub_request(:get, "http://localhost:9200/#{ilm_existence_endpoint("fluent-policy")}").
-      to_return(:status => 200, :body => "", :headers => {})
+      to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
     assert_true(ilm_policy_exists?("fluent-policy"))
   end
 
   def test_create_ilm_policy
     stub_request(:get, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}").
-      to_return(:status => 404, :body => "", :headers => {})
+      to_return(:status => 404, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
     stub_request(:put, "http://localhost:9200/#{ilm_creation_endpoint("fluent-policy")}").
       with(:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}",
          :headers => {'Content-Type'=>'application/json'}).
-      to_return(:status => 200, :body => "", :headers => {})
+      to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
     create_ilm_policy("fluent-policy")
 

data/test/plugin/test_in_elasticsearch.rb

@@ -334,7 +334,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response.to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
 
     driver(CONFIG)
@@ -350,7 +350,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     stub_request(@http_method, "http://localhost:9200/#{index_name}/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
 
     driver(CONFIG + %[index_name #{index_name}])
@@ -366,7 +366,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     stub_request(@http_method, "http://localhost:9200/#{index_name}/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
 
     driver(CONFIG + %[parse_timestamp])
@@ -385,7 +385,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     stub_request(@http_method, "http://localhost:9200/#{index_name}/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response(index_name).to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
 
     driver(CONFIG + %[parse_timestamp true
@@ -405,7 +405,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_response.to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
 
     driver(CONFIG + %[docinfo true])
@@ -425,11 +425,11 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"],\"slice\":{\"id\":0,\"max\":2}}").
       to_return(status: 200, body: sample_response.to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
     stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1000").
       with(body: "{\"sort\":[\"_doc\"],\"slice\":{\"id\":1,\"max\":2}}").
       to_return(status: 200, body: sample_response.to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
     stub_elastic_info
 
     driver(CONFIG + %[num_slices 2])
@@ -446,7 +446,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     stub_request(@http_method, "http://localhost:9200/fluentd/_search?scroll=1m&size=1").
       with(body: "{\"sort\":[\"_doc\"]}").
       to_return(status: 200, body: sample_scroll_response.to_s,
-                headers: {'Content-Type' => 'application/json'})
+                headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'})
     connection = 0
     scroll_request = stub_request(@http_method, "http://localhost:9200/_search/scroll?scroll=1m").
       with(
@@ -457,10 +457,10 @@ class ElasticsearchInputTest < Test::Unit::TestCase
     scroll_request.to_return(lambda do |req|
                                if connection <= 1
                                  {status: 200, body: sample_scroll_response_2.to_s,
-                                  headers: {'Content-Type' => 'application/json'}}
+                                  headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'}}
                                else
                                  {status: 200, body: sample_scroll_response_terminate.to_s,
-                                  headers: {'Content-Type' => 'application/json'}}
+                                  headers: {'Content-Type' => 'application/json', 'X-elastic-product' => 'Elasticsearch'}}
                                end
                              end)
     stub_request(:delete, "http://localhost:9200/_search/scroll/WomkoUKG0QPB679Ulo6TqQgh3pIGRUmrl9qXXGK3EeiQh9rbYNasTkspZQcJ01uz").