fluent-plugin-elasticsearch 3.0.2 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 73279ef661fbc0c7d8f5539b8e8524e7fce1ade17ae44e2f0ca96bd49fd047d8
-  data.tar.gz: b9b5fd6396f87231a9b430fa194da647f221083ec292cd5fa3092a5f76f57655
+  metadata.gz: 848660aa041feaaebb21f90db0da0a9e2e4b5dc912ba83c58d452607f9d1e982
+  data.tar.gz: e2c4dfedd7458cab2146b5e3939720b2fca6ce69d58be2ffbe0cc966929a2943
 SHA512:
-  metadata.gz: 7893520f2758d4c256c33c4656db6b5afb4125ff55bb18277f5edb7cee8a9cd14b7e9705f005a5c4ce6574b51548bee239cd261036f2aa3cb1f0eff24104cb65
-  data.tar.gz: 281ea7247a4d3b6ccf35a47698f632bc91fcac7f51a5b16fdb3b4a3e75a96bd518807de1c018b905e0d6cf353fe56973a06d04483f39c2decf80c67ac320429d
+  metadata.gz: 4addc8de17edeb5b1a698d21180e314789e56284576f37eed8d645d6af5bc9f7afe310fed1c16f9d63b68683d005364e0bc18324984a9029d120fcde5ee05c1d
+  data.tar.gz: 83dd202ec5ec3ad1d1cbffd14e5cdb49e90e84ec764d129a7b45fa080fa5646e56b0363241fc6cf9a0f4c7403fa43812ae45137e6499cb3014e8f84e0629d9d5

data/History.md

@@ -2,6 +2,10 @@
 
 ### [Unreleased]
 
+### 3.1.0
+- Retry obtaining Elasticsearch version (#532)
+- Fix broken id links (#530)
+
 ### 3.0.2
 -  appveyor: Remove Ruby 2.1 CI targets on AppVeyor (#524)
 - Follow removal of _routing field change on recent Elasticsearch (#523)

data/README.md

@@ -42,6 +42,7 @@ Current maintainers: @cosmo0920
   + [index_prefix](#index_prefix)
   + [templates](#templates)
   + [max_retry_putting_template](#max_retry_putting_template)
+  + [max_retry_get_es_version](#max_retry_get_es_version)
   + [request_timeout](#request_timeout)
   + [reload_connections](#reload_connections)
   + [reload_on_failure](#reload_on_failure)
@@ -67,8 +68,8 @@ Current maintainers: @cosmo0920
   + [Buffer options](#buffer-options)
   + [Hash flattening](#hash-flattening)
   + [Generate Hash ID](#generate-hash-id)
-  + [sniffer_class_name](#sniffer_class_name)
-  + [reload_after](#reload_after)
+  + [sniffer_class_name](#sniffer-class-name)
+  + [reload_after](#reload-after)
   + [validate_client_version](#validate-client-version)
   + [unrecoverable_error_types](#unrecoverable-error-types)
   + [verify_es version at startup](#verify_es_version_at_startup)
@@ -77,7 +78,7 @@ Current maintainers: @cosmo0920
   + [Dynamic configuration](#dynamic-configuration)
   + [Placeholders](#placeholders)
   + [Multi workers](#multi-workers)
-  + [log_es_400_reason](#log-es-400-reason)
+  + [log_es_400_reason](#log_es_400_reason)
 * [Troubleshooting](#troubleshooting)
   + [Cannot send events to elasticsearch](#cannot-send-events-to-elasticsearch)
   + [Cannot see detailed failure log](#cannot-see-detailed-failure-log)
@@ -431,6 +432,17 @@ Usually, booting up clustered Elasticsearch containers are much slower than laun
 max_retry_putting_template 15 # defaults to 10
 ```
 
+### max_retry_get_es_version
+
+You can specify times of retry obtaining Elasticsearch version.
+
+This is useful when Elasticsearch plugin cannot connect Elasticsearch to obtain Elasticsearch version.
+Usually, booting up clustered Elasticsearch containers are much slower than launching Fluentd container.
+
+```
+max_retry_get_es_version 17 # defaults to 15
+```
+
 ### request_timeout
 
 You can specify HTTP request timeout.

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '3.0.2'
+  s.version       = '3.1.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_error.rb

@@ -0,0 +1,5 @@
+require 'fluent/error'
+
+class Fluent::Plugin::ElasticsearchError
+  class RetryableOperationExhaustedFailure < Fluent::UnrecoverableError; end
+end

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -1,8 +1,7 @@
 require 'fluent/error'
+require_relative './elasticsearch_error'
 
 module Fluent::ElasticsearchIndexTemplate
-  class TemplateInstallationFailure < Fluent::UnrecoverableError; end
-
   def get_template(template_file)
     if !File.exists?(template_file)
       raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
@@ -29,7 +28,7 @@ module Fluent::ElasticsearchIndexTemplate
     return false
   end
 
-  def retry_install(max_retries)
+  def retry_operate(max_retries)
     return unless block_given?
     retries = 0
     begin
@@ -40,10 +39,11 @@ module Fluent::ElasticsearchIndexTemplate
       if retries < max_retries
         retries += 1
         sleep 2**retries
-        log.warn "Could not push template(s) to Elasticsearch, resetting connection and trying again. #{e.message}"
+        log.warn "Could not communicate to Elasticsearch, resetting connection and trying again. #{e.message}"
         retry
       end
-      raise TemplateInstallationFailure, "Could not push template(s) to Elasticsearch after #{retries} retries. #{e.message}"
+      raise Fluent::Plugin::ElasticsearchError::RetryableOperationExhaustedFailure,
+            "Could not communicate to Elasticsearch after #{retries} retries. #{e.message}"
     end
   end
 

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -13,6 +13,7 @@ require 'fluent/plugin/output'
 require 'fluent/event'
 require 'fluent/error'
 require_relative 'elasticsearch_constants'
+require_relative 'elasticsearch_error'
 require_relative 'elasticsearch_error_handler'
 require_relative 'elasticsearch_index_template'
 begin
@@ -106,6 +107,7 @@ EOC
     config_param :application_name, :string, :default => "default"
     config_param :templates, :hash, :default => nil
     config_param :max_retry_putting_template, :integer, :default => 10
+    config_param :max_retry_get_es_version, :integer, :default => 15
     config_param :include_tag_key, :bool, :default => false
     config_param :tag_key, :string, :default => 'tag'
     config_param :time_parse_error_tag, :string, :default => 'Fluent::ElasticsearchOutput::TimeParser.error'
@@ -170,7 +172,7 @@ EOC
       raise Fluent::ConfigError, "'max_retry_putting_template' must be positive number." if @max_retry_putting_template < 0
 
       if @template_name && @template_file
-        retry_install(@max_retry_putting_template) do
+        retry_operate(@max_retry_putting_template) do
           if @customize_template
             if @rollover_index
               raise Fluent::ConfigError, "'deflector_alias' must be provided if 'rollover_index' is set true ." if not @deflector_alias
@@ -181,7 +183,7 @@ EOC
           end
         end
       elsif @templates
-        retry_install(@max_retry_putting_template) do
+        retry_operate(@max_retry_putting_template) do
           templates_hash_install(@templates, @template_overwrite)
         end
       end
@@ -229,11 +231,8 @@ EOC
 
       @last_seen_major_version =
         if @verify_es_version_at_startup
-          begin
+          retry_operate(@max_retry_get_es_version) do
             detect_es_major_version
-          rescue
-            log.warn "Could not connect Elasticsearch or obtain version. Assuming Elasticsearch #{@default_elasticsearch_version}."
-            @default_elasticsearch_version
           end
         else
           @default_elasticsearch_version

data/test/plugin/test_out_elasticsearch.rb

@@ -350,6 +350,51 @@ class ElasticsearchOutput < Test::Unit::TestCase
     end
   end
 
+  class GetElasticsearchVersionTest < self
+    def create_driver(conf='', client_version="\"5.0\"")
+      # For request stub to detect compatibility.
+      @client_version ||= client_version
+      # Ensure original implementation existence.
+      Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
+        def detect_es_major_version
+          @_es_info ||= client.info
+          @_es_info["version"]["number"].to_i
+        end
+      CODE
+      Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
+        def client_library_version
+          #{@client_version}
+        end
+      CODE
+      Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput).configure(conf)
+    end
+
+    def test_retry_get_es_version
+      config = %{
+        host            logs.google.com
+        port            778
+        scheme          https
+        path            /es/
+        user            john
+        password        doe
+        verify_es_version_at_startup true
+        max_retry_get_es_version 3
+      }
+
+      connection_resets = 0
+      stub_request(:get, "https://john:doe@logs.google.com:778/es//").with do |req|
+        connection_resets += 1
+        raise Faraday::ConnectionFailed, "Test message"
+      end
+
+      assert_raise(Fluent::Plugin::ElasticsearchError::RetryableOperationExhaustedFailure) do
+        create_driver(config)
+      end
+
+      assert_equal(connection_resets, 4)
+    end
+  end
+
   def test_template_already_present
     config = %{
       host            logs.google.com
@@ -636,7 +681,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
       raise Faraday::ConnectionFailed, "Test message"
     end
 
-    assert_raise(Fluent::ElasticsearchIndexTemplate::TemplateInstallationFailure) do
+    assert_raise(Fluent::Plugin::ElasticsearchError::RetryableOperationExhaustedFailure) do
       driver(config)
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.0.2
+  version: 3.1.0
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-01-11 00:00:00.000000000 Z
+date: 2019-01-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -145,6 +145,7 @@ files:
 - appveyor.yml
 - fluent-plugin-elasticsearch.gemspec
 - lib/fluent/plugin/elasticsearch_constants.rb
+- lib/fluent/plugin/elasticsearch_error.rb
 - lib/fluent/plugin/elasticsearch_error_handler.rb
 - lib/fluent/plugin/elasticsearch_index_template.rb
 - lib/fluent/plugin/elasticsearch_simple_sniffer.rb