fluent-plugin-elasticsearch 4.0.5 → 4.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 81cf6831267be2a7250ab5365cf783c0680e5065ff3e3df068707ab3528bdbfb
-  data.tar.gz: ea99a64f901f9daa9cbf1ac43bc51064048765872b9c1d29a3029dd9b9f56f24
+  metadata.gz: c2f2143534244444b8377964ee5b1b116aca96f5c0d9cd57c7bec5ddb6a61983
+  data.tar.gz: 89068eedecb66309e9440b2e7e1166366ef7f2414980a7bff59d308000b78f49
 SHA512:
-  metadata.gz: 6c750270cf3051fe974ebaad5063c82cd8eb9be3afc89326ce99cd5929890f2330b2397ef2fa683f1a060f035db2266862666e09282b60ef8102cc497d2a888b
-  data.tar.gz: bee1bc70e85c8c475d943abb922531c5e668bcd7457692d006cff1fb59a669845ce8ad4a1e6a45cdca58e4a0bc63626638eeef50714bf67d6ff1c7f1a1c3c5fb
+  metadata.gz: 6b4796d25b2c88393c1a346d69f86e7998343a8224ded9c61dce2dbdf019672e50642306fdffaa5ec94b98ebaaa9a01d6a1f9a0cc1cc4863dc29f97fc79cd6cb
+  data.tar.gz: e25a9736833589adcff5663ec6e8c054bcd5b799ecdb170bbcef750e4d778fbd1f6aa1a26a0ee716b25e4b3f98d22d1523a4f0ee620a46cd8f90f12df3acfc4d

data/.travis.yml

@@ -11,7 +11,7 @@ jobs:
       os: linux
       arch: amd64
     - rvm: 2.6.3
-      gemfile: gemfiles/Gemfile.ilm
+      gemfile: gemfiles/Gemfile.without.ilm
       os: linux
       arch: amd64
     - rvm: 2.6.3

data/Gemfile

@@ -4,7 +4,8 @@ source 'https://rubygems.org'
 gemspec
 
 gem 'simplecov', require: false
-gem 'coveralls', require: false
+gem 'coveralls', ">= 0.8.0", require: false
 gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
 gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
+gem "elasticsearch-xpack"
 gem "oj"

data/History.md

@@ -2,6 +2,12 @@
 
 ### [Unreleased]
 
+### 4.0.6
+- Add fallback mechanism for handling to detect es version (#730)
+- Remove needless section (#728)
+- Handle exception if index already exists (#727)
+- Tweak test cases (#726)
+
 ### 4.0.5
 -  add logstash_dateformat as placeholder (#718)
 - Tweak travis.yml for suppressing validator warnings and add CI for Linux Arm64 architecture and macOS 10.14 (#724)

data/README.md

@@ -45,6 +45,7 @@ Current maintainers: @cosmo0920
   + [templates](#templates)
   + [max_retry_putting_template](#max_retry_putting_template)
   + [fail_on_putting_template_retry_exceed](#fail_on_putting_template_retry_exceed)
+  + [fail_on_detecting_es_version_retry_exceed](#fail_on_detecting_es_version_retry_exceed)
   + [max_retry_get_es_version](#max_retry_get_es_version)
   + [request_timeout](#request_timeout)
   + [reload_connections](#reload_connections)
@@ -99,7 +100,6 @@ Current maintainers: @cosmo0920
   + [Cannot see detailed failure log](#cannot-see-detailed-failure-log)
   + [Cannot connect TLS enabled reverse Proxy](#cannot-connect-tls-enabled-reverse-proxy)
   + [Declined logs are resubmitted forever, why?](#declined-logs-are-resubmitted-forever-why)
-  + [Suggested to increase flush_thread_count, why?](#suggested-to-increase-flush_thread_count-why)
   + [Suggested to install typhoeus gem, why?](#suggested-to-install-typhoeus-gem-why)
   + [Stopped to send events on k8s, why?](#stopped-to-send-events-on-k8s-why)
   + [Random 400 - Rejected by Elasticsearch is occured, why?](#random-400---rejected-by-elasticsearch-is-occured-why)
@@ -502,6 +502,23 @@ If you have multiple output plugin, you could use this property to do not fail o
 fail_on_putting_template_retry_exceed false # defaults to true
 ```
 
+### fail_on_detecting_es_version_retry_exceed
+
+Indicates whether to fail when `max_retry_get_es_version` is exceeded.
+If you want to use fallback mechanism for obtaining ELasticsearch version, you could use this property to do not fail on fluentd statup.
+
+```
+fail_on_detecting_es_version_retry_exceed false
+```
+
+And the following parameters should be working with:
+
+```
+verify_es_version_at_startup true
+max_retry_get_es_version 2 # greater than 0.
+default_elasticsearch_version 7 # This version is used when occurring fallback.
+```
+
 ### max_retry_get_es_version
 
 You can specify times of retry obtaining Elasticsearch version.
@@ -1445,30 +1462,6 @@ The following configuration uses label:
 </label>
 ```
 
-### Suggested to increase flush_thread_count, why?
-
-fluent-plugin-elasticsearch default behavior has a possibility to cause events traffic jam.
-When users use `flush_thread_count` = 1, ES plugin retries to send events if connection errors are disappeared.
-
-To prevent the following warning and sending events blocking, you must specify `flush_thread_count` >= 2:
-
-```log
-2018-12-24 14:32:06 +0900 [warn]: #0 To prevent events traffic jam, you should specify 2 or more 'flush_thread_count'.
-```
-
-```aconf
-<match out.elasticsearch.**>
-  @type elasticsearch
-  host localhost
-  port 9200
-  # ...
-  <buffer tag>
-    @type memory # or file
-    flush_thread_count 4
-  </buffer>
-</match>
-```
-
 ### Suggested to install typhoeus gem, why?
 
 fluent-plugin-elasticsearch doesn't depend on typhoeus gem by default.

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '4.0.5'
+  s.version       = '4.0.6'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/gemfiles/Gemfile.elasticsearch.v6

@@ -4,7 +4,7 @@ source 'https://rubygems.org'
 gemspec :path => "../"
 
 gem 'simplecov', require: false
-gem 'coveralls', require: false
+gem 'coveralls', ">= 0.8.0", require: false
 gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
 gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
 gem "elasticsearch", "~> 6.8.1"

data/gemfiles/{Gemfile.ilm → Gemfile.without.ilm}

@@ -4,8 +4,7 @@ source 'https://rubygems.org'
 gemspec :path => "../"
 
 gem 'simplecov', require: false
-gem 'coveralls', require: false
+gem 'coveralls', ">= 0.8.0", require: false
 gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
 gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
-gem "elasticsearch-xpack"
 gem "oj"

data/lib/fluent/plugin/elasticsearch_index_template.rb

@@ -58,7 +58,7 @@ module Fluent::ElasticsearchIndexTemplate
   def indexcreation(index_name, host = nil)
     client(host).indices.create(:index => index_name)
   rescue Elasticsearch::Transport::Transport::Error => e
-    if e.message =~ /"already exists"/
+    if e.message =~ /"already exists"/ || e.message =~ /resource_already_exists_exception/
       log.debug("Index #{index_name} already exists")
     else
       log.error("Error while index creation - #{index_name}: #{e.inspect}")

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -125,6 +125,7 @@ EOC
     config_param :templates, :hash, :default => nil
     config_param :max_retry_putting_template, :integer, :default => 10
     config_param :fail_on_putting_template_retry_exceed, :bool, :default => true
+    config_param :fail_on_detecting_es_version_retry_exceed, :bool, :default => true
     config_param :max_retry_get_es_version, :integer, :default => 15
     config_param :include_tag_key, :bool, :default => false
     config_param :tag_key, :string, :default => 'tag'
@@ -290,14 +291,11 @@ EOC
         raise Fluent::ConfigError, "Could not load sniffer class #{@sniffer_class_name}: #{ex}"
       end
 
-      @last_seen_major_version =
-        if @verify_es_version_at_startup && !dry_run?
-          retry_operate(@max_retry_get_es_version) do
-            detect_es_major_version
-          end
-        else
-          @default_elasticsearch_version
-        end
+      @last_seen_major_version = if major_version = handle_last_seen_es_major_version
+                                   major_version
+                                 else
+                                   @default_elasticsearch_version
+                                 end
       if @last_seen_major_version == 6 && @type_name != DEFAULT_TYPE_NAME_ES_7x
         log.info "Detected ES 6.x: ES 7.x will only accept `_doc` in type_name."
       end
@@ -421,6 +419,16 @@ EOC
       raise Fluent::ConfigError, "You must install #{@http_backend} gem. Exception: #{ex}"
     end
 
+    def handle_last_seen_es_major_version
+      if @verify_es_version_at_startup && !dry_run?
+        retry_operate(@max_retry_get_es_version, @fail_on_detecting_es_version_retry_exceed) do
+          detect_es_major_version
+        end
+      else
+        nil
+      end
+    end
+
     def detect_es_major_version
       @_es_info ||= client.info
       @_es_info["version"]["number"].to_i

data/test/plugin/test_elasticsearch_error_handler.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative '../helper'
 require 'fluent/plugin/out_elasticsearch'
 require 'fluent/plugin/elasticsearch_error_handler'
 require 'json'

data/test/plugin/test_elasticsearch_index_lifecycle_management.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative '../helper'
 require 'elasticsearch'
 require 'fluent/plugin/elasticsearch_index_lifecycle_management'
 

data/test/plugin/test_filter_elasticsearch_genid.rb

@@ -1,4 +1,4 @@
-require 'helper'
+require_relative '../helper'
 require 'date'
 require 'fluent/test/helpers'
 require 'json'

data/test/plugin/test_in_elasticsearch.rb

@@ -4,6 +4,7 @@ require 'fluent/test/helpers'
 require 'json'
 require 'fluent/test/driver/input'
 require 'flexmock/test_unit'
+require 'fluent/plugin/in_elasticsearch'
 
 class ElasticsearchInputTest < Test::Unit::TestCase
   include FlexMock::TestCase
@@ -16,7 +17,6 @@ class ElasticsearchInputTest < Test::Unit::TestCase
 
   def setup
     Fluent::Test.setup
-    require 'fluent/plugin/in_elasticsearch'
     @driver = nil
     log = Fluent::Engine.log
     log.out.logs.slice!(0, log.out.logs.length)

data/test/plugin/test_out_elasticsearch.rb

@@ -1,11 +1,12 @@
-require 'helper'
+require_relative '../helper'
 require 'date'
 require 'fluent/test/helpers'
 require 'json'
 require 'fluent/test/driver/output'
 require 'flexmock/test_unit'
+require 'fluent/plugin/out_elasticsearch'
 
-class ElasticsearchOutput < Test::Unit::TestCase
+class ElasticsearchOutputTest < Test::Unit::TestCase
   include FlexMock::TestCase
   include Fluent::Test::Helpers
 
@@ -13,7 +14,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
 
   def setup
     Fluent::Test.setup
-    require 'fluent/plugin/out_elasticsearch'
     @driver = nil
     log = Fluent::Engine.log
     log.out.logs.slice!(0, log.out.logs.length)
@@ -535,6 +535,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
          "template_name_placeholder" => ["template_name", "logstash-${mykey}"],
          "customize_template" => ["customize_template", '{"<<TAG>>":"${mykey}"}'],
          "logstash_prefix_placeholder" => ["logstash_prefix", "fluentd-${mykey}"],
+         "logstash_dateformat_placeholder" => ["logstash_dateformat", "${mykey}"],
          "deflector_alias_placeholder" => ["deflector_alias", "fluentd-${mykey}"],
          "application_name_placeholder" => ["application_name", "fluentd-${mykey}"],
         )
@@ -560,6 +561,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
          "template_name_placeholder" => ["template_name", "logstash-${tag}-%Y%m%d"],
          "customize_template" => ["customize_template", '{"<<TAG>>":"${es_index}"}'],
          "logstash_prefix_placeholder" => ["logstash_prefix", "fluentd-${es_index}-%Y%m%d"],
+         "logstash_dataformat_placeholder" => ["logstash_dateformat", "${es_index}"],
          "deflector_alias_placeholder" => ["deflector_alias", "fluentd-%Y%m%d"],
          "application_name_placeholder" => ["application_name", "fluentd-${tag}-${es_index}-%Y%m%d"],
         )
@@ -710,6 +712,60 @@ class ElasticsearchOutput < Test::Unit::TestCase
     end
   end
 
+  class GetElasticsearchVersionWithFallbackTest < self
+    def create_driver(conf='', client_version="\"5.0\"")
+      # For request stub to detect compatibility.
+      @client_version ||= client_version
+      # Ensure original implementation existence.
+      Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
+        def detect_es_major_version
+          @_es_info ||= client.info
+          @_es_info["version"]["number"].to_i
+        end
+      CODE
+      Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
+        def client_library_version
+          #{@client_version}
+        end
+      CODE
+      Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput).configure(conf)
+    end
+
+    data("Elasticsearch 5" => ["5.0", 5],
+         "Elasticsearch 6" => ["6.0", 6],
+         "Elasticsearch 7" => ["7.0", 7],
+         "Elasticsearch 8" => ["8.0", 8])
+    def test_retry_get_es_version_without_fail_on_detecting_es_version_retry_exceeded(data)
+      client_version, es_major_version = data
+      config = %{
+        host            logs.google.com
+        port            778
+        scheme          https
+        path            /es/
+        user            john
+        password        doe
+        verify_es_version_at_startup true
+        max_retry_get_es_version 2
+        fail_on_detecting_es_version_retry_exceed false
+        default_elasticsearch_version #{es_major_version}
+        @log_level info
+      }
+
+      connection_resets = 0
+      stub_request(:get, "https://logs.google.com:778/es//").
+        with(basic_auth: ['john', 'doe']) do |req|
+        connection_resets += 1
+        raise Faraday::ConnectionFailed, "Test message"
+      end
+
+      d = create_driver(config, client_version)
+
+      assert_equal es_major_version, d.instance.default_elasticsearch_version
+      assert_equal 3, connection_resets
+      assert_equal es_major_version, d.instance.instance_variable_get(:@last_seen_major_version)
+    end
+  end
+
   def test_template_already_present
     config = %{
       host            logs.google.com

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -1,8 +1,9 @@
-require 'helper'
+require_relative '../helper'
 require 'date'
 require 'fluent/test/helpers'
 require 'fluent/test/driver/output'
 require 'flexmock/test_unit'
+require 'fluent/plugin/out_elasticsearch_dynamic'
 
 class ElasticsearchOutputDynamic < Test::Unit::TestCase
   include FlexMock::TestCase
@@ -12,7 +13,6 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
 
   def setup
     Fluent::Test.setup
-    require 'fluent/plugin/out_elasticsearch_dynamic'
     @driver = nil
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.0.5
+  version: 4.0.6
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-02 00:00:00.000000000 Z
+date: 2020-03-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -149,7 +149,7 @@ files:
 - appveyor.yml
 - fluent-plugin-elasticsearch.gemspec
 - gemfiles/Gemfile.elasticsearch.v6
-- gemfiles/Gemfile.ilm
+- gemfiles/Gemfile.without.ilm
 - lib/fluent/log-ext.rb
 - lib/fluent/plugin/default-ilm-policy.json
 - lib/fluent/plugin/elasticsearch_constants.rb