fluent-plugin-elasticsearch 3.4.2 → 3.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3882c4cbe493e8dcbba56ddd5af9bd1ecdc85bea95e19bbe4f4fd302b5cc5edf
-  data.tar.gz: a184602aae1d5665879cb2cbc01910068af76f4579af630a7b466a9d495c62ed
+  metadata.gz: 599f144cfd094f25416002686c3f4c11b693c3f5a28c33d18ddd670639d95b6f
+  data.tar.gz: fcfcfaf1c73a5338740934d887273399a7b4396a2246f392ceb779281a7da9d7
 SHA512:
-  metadata.gz: a6e9fb90f39b99907ec3df6dc1770d9eaa449cbee292669d1de7b13d2a38488683c4820a755553410c81fbd6e665ea573750d790b8bf7ad91304213baddc6b49
-  data.tar.gz: f047c3a93f5bc9c3609b6c6c2505730a11c7b5d7f4410fe0e7f950a3dc5e995336f541023997a7a3dd28149872f3af457c796cf37c438e0c93f72bcb8eed11d6
+  metadata.gz: 0c7a7176b04bbb770ad1ad9fd7f67a1f4ff8cdc7b4206fca5e1e677518c00ffa2329a1ae5ebeb1dd25e2d5324edd8df5e4c92a90c59a337b7ed2e68827aef4b6
+  data.tar.gz: 69f91ef1ecbd148da6feac924f1d8c86668aa50191a76eccd53d9968e5ae47f6b33039ea1c693784fe89c688f0e4ee772077e9085ac511032b1490bdf3565706

data/History.md

@@ -2,6 +2,9 @@
 
 ### [Unreleased]
 
+### 3.4.3
+- Add fail_on_putting_template_retry_exceed config (#579)
+
 ### 3.4.2
 - Comparing DEFAULT_TYPE_NAME_ES_7x to target_type instead of type_name (#573)
 

data/README.md

@@ -43,6 +43,7 @@ Current maintainers: @cosmo0920
   + [index_prefix](#index_prefix)
   + [templates](#templates)
   + [max_retry_putting_template](#max_retry_putting_template)
+  + [fail_on_putting_template_retry_exceed](#fail_on_putting_template_retry_exceed)
   + [max_retry_get_es_version](#max_retry_get_es_version)
   + [request_timeout](#request_timeout)
   + [reload_connections](#reload_connections)
@@ -450,6 +451,15 @@ Usually, booting up clustered Elasticsearch containers are much slower than laun
 max_retry_putting_template 15 # defaults to 10
 ```
 
+### fail_on_putting_template_retry_exceed
+
+Indicates whether to fail when `max_retry_putting_template` is exceeded.
+If you have multiple output plugin, you could use this property to do not fail on fluentd statup.
+
+```
+fail_on_putting_template_retry_exceed false # defaults to true
+```
+
 ### max_retry_get_es_version
 
 You can specify times of retry obtaining Elasticsearch version.

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '3.4.2'
+  s.version       = '3.4.3'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -28,7 +28,7 @@ module Fluent::ElasticsearchIndexTemplate
     return false
   end
 
-  def retry_operate(max_retries)
+  def retry_operate(max_retries, fail_on_retry_exceed = true)
     return unless block_given?
     retries = 0
     begin
@@ -42,8 +42,10 @@ module Fluent::ElasticsearchIndexTemplate
         log.warn "Could not communicate to Elasticsearch, resetting connection and trying again. #{e.message}"
         retry
       end
+      message = "Could not communicate to Elasticsearch after #{retries} retries. #{e.message}"
+      log.warn message
       raise Fluent::Plugin::ElasticsearchError::RetryableOperationExhaustedFailure,
-            "Could not communicate to Elasticsearch after #{retries} retries. #{e.message}"
+            message if fail_on_retry_exceed
     end
   end
 

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -109,6 +109,7 @@ EOC
     config_param :application_name, :string, :default => "default"
     config_param :templates, :hash, :default => nil
     config_param :max_retry_putting_template, :integer, :default => 10
+    config_param :fail_on_putting_template_retry_exceed, :bool, :default => true
     config_param :max_retry_get_es_version, :integer, :default => 15
     config_param :include_tag_key, :bool, :default => false
     config_param :tag_key, :string, :default => 'tag'
@@ -185,7 +186,7 @@ EOC
 
       if !Fluent::Engine.dry_run_mode
         if @template_name && @template_file
-          retry_operate(@max_retry_putting_template) do
+          retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
             if @customize_template
               if @rollover_index
                 raise Fluent::ConfigError, "'deflector_alias' must be provided if 'rollover_index' is set true ." if not @deflector_alias
@@ -196,7 +197,7 @@ EOC
             end
           end
         elsif @templates
-          retry_operate(@max_retry_putting_template) do
+          retry_operate(@max_retry_putting_template, @fail_on_putting_template_retry_exceed) do
             templates_hash_install(@templates, @template_overwrite)
           end
         end

data/test/plugin/test_out_elasticsearch.rb

@@ -679,7 +679,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     end
   end
 
-  def test_template_retry_install
+  def test_template_retry_install_fails
     cwd = File.dirname(__FILE__)
     template_file = File.join(cwd, 'test_template.json')
 
@@ -709,6 +709,35 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(connection_resets, 4)
   end
 
+  def test_template_retry_install_does_not_fail
+    cwd = File.dirname(__FILE__)
+    template_file = File.join(cwd, 'test_template.json')
+
+    config = %{
+      host            logs.google.com
+      port            778
+      scheme          https
+      path            /es/
+      user            john
+      password        doe
+      template_name   logstash
+      template_file   #{template_file}
+      max_retry_putting_template 3
+      fail_on_putting_template_retry_exceed false
+    }
+
+    connection_resets = 0
+    # check if template exists
+    stub_request(:get, "https://john:doe@logs.google.com:778/es//_template/logstash").with do |req|
+      connection_resets += 1
+      raise Faraday::ConnectionFailed, "Test message"
+    end
+
+    driver(config)
+
+    assert_equal(connection_resets, 4)
+  end
+
   def test_templates_create
     cwd = File.dirname(__FILE__)
     template_file = File.join(cwd, 'test_template.json')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.4.2
+  version: 3.4.3
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-14 00:00:00.000000000 Z
+date: 2019-04-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -179,7 +179,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.1
 signing_key: 
 specification_version: 4
 summary: Elasticsearch output plugin for Fluent event collector