fluent-plugin-elasticsearch 4.2.2 → 4.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d54b3f207af54f6fb5d3119f2d6d649478e25471fda03c56e00f9206ae72914e
-  data.tar.gz: 82eea8ed3938919f6860582698ce1253214531c806c132723e8845adfb45e2e0
+  metadata.gz: 5aaeb8e93dcb477b3c2d83bc469cd327068aa59f9dc073e8578cea6320b5a4d4
+  data.tar.gz: 3b1f76ac44342ce51b53dce4d4b15be77744c21f130b6bd2cd5eee0de4dae4b2
 SHA512:
-  metadata.gz: c4fd2785f716d97cb8c44845c51f933f7d0ee69fe3e2911f85bf621d043015e815639350e363ce42866b224905fc20ae12b3666d302e5f1f07937f047123b20d
-  data.tar.gz: 15d34dc457c5a98b77f2142b271d3f4ef5f611e38caeb4f225f12dcfc2c3771dea0153bae3ebe99a83cc71c6e414d67e39fea1d2fa1fa430049f90839c677578
+  metadata.gz: 8605eeb221ebf030b440865cb049b092ba3a5f57264226f22edacbdb07844301820a8d96e230a6593cdd29ec1fb0febb4f0aa8b7225ec80802f5f0f5b1a8a82a
+  data.tar.gz: dcc159325045a0609935c92b53085e564109bcc3d3eb75946226f0c5ac4d7899e79b9480aede76545378c6a0c6927ee92505f5c1aa1801b89d11fa857650964d

data/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,37 @@
+---
+name:  Bug Report
+about: Create a report to help us improve. If you have questions about ES plugin on kubernetes, please direct these to https://discuss.kubernetes.io/ before sumbit kubernetes related issue.
+
+---
+
+(check apply)
+- [ ] read [the contribution guideline](https://github.com/uken/fluent-plugin-elasticsearch/blob/master/CONTRIBUTING.md)
+- [ ] (optional) already reported 3rd party upstream repository or mailing list if you use k8s addon or helm charts.
+
+#### Problem
+
+...
+
+#### Steps to replicate
+
+Either clone and modify https://gist.github.com/pitr/9a518e840db58f435911
+
+**OR**
+
+Provide example config and message
+
+#### Expected Behavior or What you need to ask
+
+...
+
+#### Using Fluentd and ES plugin versions
+
+* OS version
+* Bare Metal or within Docker or Kubernetes or others?
+* Fluentd v0.12 or v0.14/v1.0
+  * paste result of ``fluentd --version`` or ``td-agent --version``
+* ES plugin 3.x.y/2.x.y or 1.x.y
+  * paste boot log of fluentd or td-agent
+  * paste result of ``fluent-gem list``, ``td-agent-gem list`` or your Gemfile.lock
+* ES version (optional)
+* ES template(s) (optional)

data/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,24 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+
+---
+
+(check apply)
+- [ ] read [the contribution guideline](https://github.com/uken/fluent-plugin-elasticsearch/blob/master/CONTRIBUTING.md)
+
+**Is your feature request related to a problem? Please describe.**
+
+<!-- A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] -->
+
+**Describe the solution you'd like**
+
+<!-- A clear and concise description of what you want to happen. -->
+
+**Describe alternatives you've considered**
+
+<!-- A clear and concise description of any alternative solutions or features you've considered. -->
+
+**Additional context**
+
+<!-- Add any other context or screenshots about the feature request here. -->

data/.github/workflows/issue-auto-closer.yml

@@ -9,4 +9,4 @@ jobs:
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         issue-close-message: "@${issue.user.login} this issue was automatically closed because it did not follow the issue template."
-        issue-pattern: "(.*Problem.*)|(.*Expected Behavior or What you need to ask.*)|(.*Using Fluentd and ES plugin versions.*)"
+        issue-pattern: "(.*Problem.*)|(.*Expected Behavior or What you need to ask.*)|(.*Using Fluentd and ES plugin versions.*)|(.*Is your feature request related to a problem? Please describe.*)|(.*Describe the solution you'd like.*)|(.*Describe alternatives you've considered.*)"

data/History.md

@@ -2,6 +2,13 @@
 
 ### [Unreleased]
 
+### 4.3.0
+- Add cloud_id & cloud_auth settings (#850)
+- Fix failing tests on es6 client (#848)
+- feat: retry on transport errors (#846)
+- fix typo (#838)
+- Remove unnecessary nil check (#826)
+
 ### 4.2.2
 - Remove unnecessary nil check (#826)
 - Fix ILM rollover index template pattern to apply index_separator (#825)

data/README.ElasticsearchGenID.md

@@ -17,7 +17,7 @@
 In your Fluentd configuration, use `@type elasticsearch_genid`. Additional configuration is optional, default values would look like this:
 
 ```
-<source>
+<filter>
   @type elasticsearch_genid
   hash_id_key _hash
   include_tag_in_seed false
@@ -27,7 +27,7 @@ In your Fluentd configuration, use `@type elasticsearch_genid`. Additional confi
   record_keys []
   separator _
   hash_type sha1
-</match>
+</filter>
 ```
 
 ## Configuration

data/README.md

@@ -19,6 +19,8 @@ Current maintainers: @cosmo0920
 * [Configuration](#configuration)
   + [host](#host)
   + [port](#port)
+  + [cloud_id](#cloud_id)
+  + [cloud_auth](#cloud_auth)
   + [emit_error_for_missing_id](#emit_error_for_missing_id)
   + [hosts](#hosts)
   + [user, password, path, scheme, ssl_verify](#user-password-path-scheme-ssl_verify)
@@ -187,6 +189,26 @@ port 9201 # defaults to 9200
 
 You can specify Elasticsearch port by this parameter.
 
+### cloud_id
+
+```
+cloud_id test-dep:ZXVyb3BlLXdlc3QxLmdjcC5jbG91ZC5lcy5pbyRiYZTA1Ng== 
+```
+
+You can specify Elasticsearch cloud_id by this parameter.
+
+If you specify `cloud_id` option then `cloud_auth` option is required.
+If you specify `cloud_id` option, `host`, `port`, `user` and `password` options are ignored.
+
+### cloud_auth
+
+```
+cloud_auth 'elastic:slkjdaooewkd87iqQ2O8EQYV'
+```
+
+You can specify Elasticsearch cloud_auth by this parameter.
+
+
 ### emit_error_for_missing_id
 
 ```
@@ -501,7 +523,7 @@ Specify the application name for the rollover index to be created.
 application_name default # defaults to "default"
 ```
 
-If [enable_ilm](#enable_ilm is set, then this parameter will be in effect otherwise ignored.
+If [enable_ilm](#enable_ilm) is set, then this parameter will be in effect otherwise ignored.
 
 ### template_overwrite
 
@@ -1143,6 +1165,8 @@ And yet another option is described in Dynamic Configuration section.
 
 ### Dynamic configuration
 
+**NOTE**: *`out_elasticsearch_dynamic` will be planned to be marked as deprecated.* Please don't use the new Fluentd configuration. This plugin is maintained for backward compatibility.
+
 If you want configurations to depend on information in messages, you can use `elasticsearch_dynamic`. This is an experimental variation of the Elasticsearch plugin allows configuration values to be specified in ways such as the below:
 
 ```

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '4.2.2'
+  s.version       = '4.3.0'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -32,12 +32,13 @@ module Fluent::ElasticsearchIndexTemplate
     return false
   end
 
-  def retry_operate(max_retries, fail_on_retry_exceed = true)
+  def retry_operate(max_retries, fail_on_retry_exceed = true, catch_trasport_exceptions = true)
     return unless block_given?
     retries = 0
+    transport_errors = Elasticsearch::Transport::Transport::Errors.constants.map{ |c| Elasticsearch::Transport::Transport::Errors.const_get c } if catch_trasport_exceptions
     begin
       yield
-    rescue *client.transport.host_unreachable_exceptions, Timeout::Error => e
+    rescue *client.transport.host_unreachable_exceptions, *transport_errors, Timeout::Error => e
       @_es = nil
       @_es_info = nil
       if retries < max_retries

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -77,6 +77,8 @@ module Fluent::Plugin
     config_param :port, :integer, :default => 9200
     config_param :user, :string, :default => nil
     config_param :password, :string, :default => nil, :secret => true
+    config_param :cloud_id, :string, :default => nil
+    config_param :cloud_auth, :string, :default => nil
     config_param :path, :string, :default => nil
     config_param :scheme, :enum, :list => [:https, :http], :default => :http
     config_param :hosts, :string, :default => nil
@@ -171,6 +173,7 @@ EOC
     config_param :ilm_policy_overwrite, :bool, :default => false
     config_param :truncate_caches_interval, :time, :default => nil
     config_param :use_legacy_template, :bool, :default => true
+    config_param :catch_transport_exception_on_retry, :bool, :default => true
 
     config_section :metadata, param_name: :metainfo, multi: false do
       config_param :include_chunk_id, :bool, :default => false
@@ -261,7 +264,9 @@ EOC
           verify_ilm_working if @enable_ilm
         end
         if @templates
-          retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
+          retry_operate(@max_retry_putting_template,
+                        @fail_on_putting_template_retry_exceed,
+                        @catch_transport_exception_on_retry) do
             templates_hash_install(@templates, @template_overwrite)
           end
         end
@@ -291,8 +296,14 @@ EOC
         @dump_proc = Yajl.method(:dump)
       end
 
+      raise Fluent::ConfigError, "`cloud_auth` must be present if `cloud_id` is present" if @cloud_id && @cloud_auth.nil?
       raise Fluent::ConfigError, "`password` must be present if `user` is present" if @user && @password.nil?
 
+      if @cloud_auth
+        @user = @cloud_auth.split(':', -1)[0]
+        @password = @cloud_auth.split(':', -1)[1]
+      end
+
       if @user && m = @user.match(/%{(?<user>.*)}/)
         @user = URI.encode_www_form_component(m["user"])
       end
@@ -470,7 +481,9 @@ EOC
 
     def handle_last_seen_es_major_version
       if @verify_es_version_at_startup && !dry_run?
-        retry_operate(@max_retry_get_es_version, @fail_on_detecting_es_version_retry_exceed) do
+        retry_operate(@max_retry_get_es_version,
+                      @fail_on_detecting_es_version_retry_exceed,
+                      @catch_transport_exception_on_retry) do
           detect_es_major_version
         end
       else
@@ -555,7 +568,17 @@ EOC
       return Time.at(event_time).to_datetime
     end
 
+    def cloud_client
+      Elasticsearch::Client.new(
+        cloud_id: @cloud_id,
+        user: @user,
+        password: @password
+      )
+    end
+
     def client(host = nil, compress_connection = false)
+      return cloud_client if @cloud_id
+
       # check here to see if we already have a client connection for the given host
       connection_options = get_connection_options(host)
 
@@ -989,7 +1012,9 @@ EOC
             log.debug("Template #{template_name} already exists (cached)")
           end
         else
-          retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
+          retry_operate(@max_retry_putting_template,
+                        @fail_on_putting_template_retry_exceed,
+                        @catch_transport_exception_on_retry) do
             if customize_template
               template_custom_install(template_name, @template_file, @template_overwrite, customize_template, @enable_ilm, deflector_alias, ilm_policy_id, host, target_index, @index_separator)
             else

data/test/plugin/test_elasticsearch_fallback_selector.rb

@@ -58,6 +58,7 @@ class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
       with_transporter_log true
       reload_connections true
       reload_after 10
+      catch_transport_exception_on_retry false # For fallback testing
     ]
     assert_raise(Elasticsearch::Transport::Transport::Errors::NotFound) do
       driver(config)

data/test/plugin/test_elasticsearch_index_lifecycle_management.rb

@@ -11,6 +11,9 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
     rescue LoadError
       omit "ILM testcase needs elasticsearch-xpack gem."
     end
+    if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.4.0")
+      omit "elastisearch-ruby v7.4.0 or later is needed for ILM."
+    end
     Fluent::Plugin::ElasticsearchIndexLifecycleManagement.module_eval(<<-CODE)
       def client
         Elasticsearch::Client.new url: 'localhost:9200'

data/test/plugin/test_out_elasticsearch.rb

@@ -322,6 +322,19 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     assert_equal true, instance.client(nil, compressable).transport.options[:compression]
   end
 
+  test 'check configure cloud_id based client' do
+
+    config = %{
+      cloud_id "name:bG9jYWxob3N0JGFiY2QkZWZnaA=="
+      cloud_auth "some:auth"
+    }
+    instance = driver(config).instance
+
+    assert_equal "name:bG9jYWxob3N0JGFiY2QkZWZnaA==", instance.cloud_id
+    assert_equal "some", instance.user
+    assert_equal "auth", instance.password
+  end
+
   test 'configure Content-Type' do
     config = %{
       content_type application/x-ndjson
@@ -383,7 +396,13 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     data("legacy_template" => [true, "_template"],
          "new_template"    => [false, "_index_template"])
     test 'valid configuration of index lifecycle management' do |data|
+      if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.4.0")
+        omit "elastisearch-ruby v7.4.0 or later is needed for ILM."
+      end
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = File.join(cwd, 'test_template.json')
 
@@ -419,7 +438,13 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     data("legacy_template" => [true, "_template"],
          "new_template"    => [false, "_index_template"])
     test 'valid configuration of overwriting ilm_policy' do |data|
+      if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.4.0")
+        omit "elastisearch-ruby v7.4.0 or later is needed for ILM."
+      end
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = File.join(cwd, 'test_template.json')
 
@@ -823,6 +848,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_template_already_present(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     config = %{
       host            logs.google.com
       port            777
@@ -853,6 +881,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_template_create(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -894,6 +925,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_template_create_with_rollover_index_and_template_related_placeholders(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -959,6 +993,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_template_create_with_rollover_index_and_template_related_placeholders_with_truncating_caches(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -1034,12 +1071,18 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       rescue LoadError
         omit "ILM testcase needs elasticsearch-xpack gem."
       end
+      if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.4.0")
+        omit "elastisearch-ruby v7.4.0 or later is needed for ILM."
+      end
     end
 
     data("legacy_template" => [true, "_template"],
          "new_template"    => [false, "_index_template"])
     def test_template_create_with_rollover_index_and_default_ilm(data)
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = if use_legacy_template_flag
                         File.join(cwd, 'test_template.json')
@@ -1120,6 +1163,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
          "new_template"    => [false, "_index_template"])
     def test_template_create_with_rollover_index_and_default_ilm_on_logstash_format(data)
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = if use_legacy_template_flag
                         File.join(cwd, 'test_template.json')
@@ -1209,6 +1255,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
          "new_template"    => [false, "_index_template"])
     def test_template_create_with_rollover_index_and_default_ilm_and_ilm_policy_overwrite(data)
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = if use_legacy_template_flag
                         File.join(cwd, 'test_template.json')
@@ -1316,6 +1365,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
          "new_template"    => [false, "_index_template"])
     def test_template_create_with_rollover_index_and_default_ilm_with_empty_index_date_pattern(data)
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = if use_legacy_template_flag
                         File.join(cwd, 'test_template.json')
@@ -1396,6 +1448,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
          "new_template"    => [false, "_index_template"])
     def test_template_create_with_rollover_index_and_custom_ilm(data)
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = if use_legacy_template_flag
                         File.join(cwd, 'test_template.json')
@@ -1477,6 +1532,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
          "new_template"    => [false, "_index_template"])
     def test_template_create_with_rollover_index_and_ilm_policies_and_placeholderstest_template_create_with_rollover_index_and_ilm_policies_and_placeholders(data)
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = if use_legacy_template_flag
                         File.join(cwd, 'test_template.json')
@@ -1565,6 +1623,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
            "new_template"    => [false, "_index_template"])
       def test_tag_placeholder(data)
         use_legacy_template_flag, endpoint = data
+        if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+          omit "elastisearch-ruby v7.8.0 or later is needed."
+        end
         cwd = File.dirname(__FILE__)
         template_file = if use_legacy_template_flag
                           File.join(cwd, 'test_template.json')
@@ -1652,6 +1713,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
            "new_template"    => [false, "_index_template"])
       def test_tag_placeholder_with_multiple_policies(data)
         use_legacy_template_flag, endpoint = data
+        if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+          omit "elastisearch-ruby v7.8.0 or later is needed."
+        end
         cwd = File.dirname(__FILE__)
         template_file = if use_legacy_template_flag
                           File.join(cwd, 'test_template.json')
@@ -1740,6 +1804,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
          "new_template"    => [false, "_index_template"])
     def test_template_create_with_rollover_index_and_default_ilm_and_placeholders(data)
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = if use_legacy_template_flag
                         File.join(cwd, 'test_template.json')
@@ -1867,6 +1934,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
          "new_template"    => [false, "_index_template"])
     def test_template_create_with_rollover_index_and_default_ilm_and_placeholders_and_index_separator(data)
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = if use_legacy_template_flag
                         File.join(cwd, 'test_template.json')
@@ -1956,6 +2026,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
          "new_template"    => [false, "_index_template"])
     def test_template_create_with_rollover_index_and_default_ilm_and_custom_and_time_placeholders(data)
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = if use_legacy_template_flag
                         File.join(cwd, 'test_template.json')
@@ -2050,6 +2123,10 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_custom_template_create(data)
     use_legacy_template_flag, endpoint = data
+
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_alias_template.json')
@@ -2092,6 +2169,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_custom_template_create_with_customize_template_related_placeholders(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_alias_template.json')
@@ -2142,6 +2222,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_custom_template_installation_for_host_placeholder(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -2188,6 +2271,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_custom_template_with_rollover_index_create(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_alias_template.json')
@@ -2246,6 +2332,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_custom_template_with_rollover_index_create_and_deflector_alias(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_alias_template.json')
@@ -2305,6 +2394,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_custom_template_with_rollover_index_create_with_logstash_format(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_alias_template.json')
@@ -2373,12 +2465,18 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
       rescue LoadError
         omit "ILM testcase needs elasticsearch-xpack gem."
       end
+      if Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.4.0")
+        omit "elastisearch-ruby v7.4.0 or later is needed."
+      end
     end
 
     data("legacy_template" => [true, "_template"],
          "new_template"    => [false, "_index_template"])
     def test_custom_template_with_rollover_index_create_and_default_ilm(data)
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = if use_legacy_template_flag
                         File.join(cwd, 'test_alias_template.json')
@@ -2463,6 +2561,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
          "new_template"    => [false, "_index_template"])
     def test_custom_template_with_rollover_index_create_and_default_ilm_and_ilm_policy_overwrite(data)
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = if use_legacy_template_flag
                         File.join(cwd, 'test_alias_template.json')
@@ -2577,6 +2678,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
          "new_template"    => [false, "_index_template"])
     def test_custom_template_with_rollover_index_create_and_default_ilm_and_placeholders(data)
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = if use_legacy_template_flag
                         File.join(cwd, 'test_alias_template.json')
@@ -2720,6 +2824,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
          "new_template"    => [false, "_index_template"])
     def test_custom_template_with_rollover_index_create_and_custom_ilm(data)
       use_legacy_template_flag, endpoint = data
+      if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+        omit "elastisearch-ruby v7.8.0 or later is needed."
+      end
       cwd = File.dirname(__FILE__)
       template_file = if use_legacy_template_flag
                         File.join(cwd, 'test_alias_template.json')
@@ -2798,6 +2905,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_template_overwrite(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -2840,6 +2950,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_custom_template_overwrite(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -2883,6 +2996,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_custom_template_with_rollover_index_overwrite(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -2969,6 +3085,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_template_create_for_host_placeholder(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -3017,6 +3136,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_template_retry_install_fails(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -3052,10 +3174,54 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
     assert_equal(4, connection_resets)
   end
 
+  transport_errors_handled_separately = [Elasticsearch::Transport::Transport::Errors::NotFound]
+  transport_errors = Elasticsearch::Transport::Transport::Errors.constants.map { |err| [err, Elasticsearch::Transport::Transport::Errors.const_get(err)]  }
+  transport_errors_hash = Hash[transport_errors.select { |err| !transport_errors_handled_separately.include?(err[1]) } ]
+
+  data(transport_errors_hash)
+  def test_template_retry_transport_errors(error)
+    endpoint, use_legacy_template_flag = if Gem::Version.create(::Elasticsearch::Transport::VERSION) >= Gem::Version.create("7.8.0")
+                 ["_index_template".freeze, false]
+               else
+                 ["_template".freeze, true]
+               end
+    cwd = File.dirname(__FILE__)
+    template_file = File.join(cwd, 'test_index_template.json')
+
+    config = %{
+      host            logs.google.com
+      port            778
+      scheme          https
+      path            /es/
+      user            john
+      password        doe
+      template_name   logstash
+      template_file   #{template_file}
+      max_retry_putting_template 0
+      use_legacy_template #{use_legacy_template_flag}
+    }
+
+    retries = 0
+    stub_request(:get, "https://logs.google.com:778/es//#{endpoint}/logstash")
+      .with(basic_auth: ['john', 'doe']) do |req|
+      retries += 1
+      raise error
+    end
+
+    assert_raise(Fluent::Plugin::ElasticsearchError::RetryableOperationExhaustedFailure) do
+      driver(config)
+    end
+
+    assert_equal(1, retries)
+  end
+
   data("legacy_template" => [true, "_template"],
        "new_template"    => [false, "_index_template"])
   def test_template_retry_install_does_not_fail(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -3094,6 +3260,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_templates_create(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -3148,6 +3317,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_templates_overwrite(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -3202,6 +3374,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_templates_are_also_used(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')
@@ -3258,6 +3433,9 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
        "new_template"    => [false, "_index_template"])
   def test_templates_can_be_partially_created_if_error_occurs(data)
     use_legacy_template_flag, endpoint = data
+    if !use_legacy_template_flag && Gem::Version.create(::Elasticsearch::Transport::VERSION) < Gem::Version.create("7.8.0")
+      omit "elastisearch-ruby v7.8.0 or later is needed."
+    end
     cwd = File.dirname(__FILE__)
     template_file = if use_legacy_template_flag
                       File.join(cwd, 'test_template.json')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.2.2
+  version: 4.3.0
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-16 00:00:00.000000000 Z
+date: 2020-12-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -135,6 +135,8 @@ extra_rdoc_files: []
 files:
 - ".coveralls.yml"
 - ".editorconfig"
+- ".github/ISSUE_TEMPLATE/bug_report.md"
+- ".github/ISSUE_TEMPLATE/feature_request.md"
 - ".github/workflows/issue-auto-closer.yml"
 - ".github/workflows/linux.yml"
 - ".github/workflows/macos.yml"