fluent-plugin-elasticsearch 3.3.3 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d075f2b8fe2deb8579ce232104c2e497eb6aee6703b28f0288728b2f2622c23
-  data.tar.gz: 580c94f146d2039d242cc0e02487bc771e12e0f9ac885d634ea4fc169c6e577b
+  metadata.gz: 4e8eed03eaeddf3765e1ac07f5038cd026bacbc7f86d63b286af4da186c7d04f
+  data.tar.gz: 17d90e2717897e2a021629b4414c475954ff0100c16b9d4db2526e8bd4ffc084
 SHA512:
-  metadata.gz: d98b07c8fe8e0c13d290e50957c9f2f6f6d988173bc3dbe3a6f36bedf546531f85639d3fce637295eefb14d5c3726a77e3b91fd418b451080caae03399177df8
-  data.tar.gz: 3cc3ab52cb741c020568fd55e82d496805b5a66b2d54ea6a9cf09917413099934f17bc6d69fce8b9ef431f7b0dde6662d27d9649d6ecd4330dce55fae932364d
+  metadata.gz: 94d36c2a7804887d9f974e26d55a2abbd8ade8e042126f5f6b7bbc586db0ff63fe6195feb8a51d5c610e8ffbc9274d76a886290f49af10ce0ff005d72758fb81
+  data.tar.gz: 0b3bfaf992cfebd919cdc526c52e5b3200cc70cc424bce51a197cc7915890e9b495904ce367cdb329df69b182c6928675d6872f9ce4e579d55cb2f9e5b53304d

data/History.md

@@ -2,6 +2,11 @@
 
 ### [Unreleased]
 
+### 3.4.0
+- Check exclusive feature on #configure (#569)
+- Modify FAQ for highly load k8s EFK stack (#566)
+- Add FAQ for high load k8s EFK stack (#564)
+
 ### 3.3.3
 - Add unit test for exception message (#563)
 - Add ignore_exceptions config (#562)

data/README.md

@@ -91,6 +91,7 @@ Current maintainers: @cosmo0920
   + [Declined logs are resubmitted forever, why?](#declined-logs-are-resubmitted-forever-why)
   + [Suggested to increase flush_thread_count, why?](#suggested-to-increase-flush_thread_count-why)
   + [Suggested to install typhoeus gem, why?](#suggested-to-install-typhoeus-gem-why)
+  + [Stopped to send events on k8s, why?](#stopped-to-send-events-on-k8s-why)
 * [Contact](#contact)
 * [Contributing](#contributing)
 * [Running tests](#running-tests)
@@ -1330,6 +1331,30 @@ td-agent-gem install typhoeus
 
 In more detail, please refer to [the official plugin management document](https://docs.fluentd.org/v1.0/articles/plugin-management).
 
+### Stopped to send events on k8s, why?
+
+fluent-plugin-elasticsearch reloads connection after 10000 requests. (Not correspond to events counts because ES plugin uses bulk API.)
+
+This functionality which is originated from elasticsaearch-ruby gem is enabled by default.
+
+Sometimes this reloading functionality bothers users to send events with ES plugin.
+
+On k8s platform, users sometimes shall specify the following settings:
+
+```aconf
+reload_connections false
+reconnect_on_error true
+reload_on_failure true
+```
+
+If you use [fluentd-kubernetes-daemonset](https://github.com/fluent/fluentd-kubernetes-daemonset), you can specify them with environment variables:
+
+* `FLUENT_ELASTICSEARCH_RELOAD_CONNECTIONS` as `false`
+* `FLUENT_ELASTICSEARCH_RECONNECT_ON_ERROR` as `true`
+* `FLUENT_ELASTICSEARCH_RELOAD_ON_FAILURE` as `true`
+
+This issue had been reported at [#525](https://github.com/uken/fluent-plugin-elasticsearch/issues/525).
+
 ## Contact
 
 If you have a question, [open an Issue](https://github.com/uken/fluent-plugin-elasticsearch/issues).

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '3.3.3'
+  s.version       = '3.4.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -176,6 +176,12 @@ EOC
 
       raise Fluent::ConfigError, "'max_retry_putting_template' must be positive number." if @max_retry_putting_template < 0
 
+      # Raise error when using host placeholders and template features at same time.
+      valid_host_placeholder = placeholder?(:host_placeholder, @host)
+      if valid_host_placeholder && (@template_name && @template_file || @templates)
+        raise Fluent::ConfigError, "host placeholder and template installation are exclusive features."
+      end
+
       if !Fluent::Engine.dry_run_mode
         if @template_name && @template_file
           retry_operate(@max_retry_putting_template) do
@@ -289,6 +295,15 @@ EOC
       end.compact
     end
 
+    def placeholder?(name, param)
+      begin
+        placeholder_validate!(name, param)
+        true
+      rescue Fluent::ConfigError
+        false
+      end
+    end
+
     def backend_options
       case @http_backend
       when :excon

data/test/plugin/test_out_elasticsearch.rb

@@ -659,6 +659,26 @@ class ElasticsearchOutput < Test::Unit::TestCase
     }
   end
 
+  def test_template_installation_exclusive_for_host_placeholder
+    cwd = File.dirname(__FILE__)
+    template_file = File.join(cwd, 'test_template.json')
+
+    config = %{
+      host            logs-${tag}.google.com
+      port            777
+      scheme          https
+      path            /es/
+      user            john
+      password        doe
+      template_name   logstash
+      template_file   #{template_file}
+    }
+
+    assert_raise(Fluent::ConfigError) do
+      driver(config)
+    end
+  end
+
   def test_template_retry_install
     cwd = File.dirname(__FILE__)
     template_file = File.join(cwd, 'test_template.json')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.3.3
+  version: 3.4.0
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-20 00:00:00.000000000 Z
+date: 2019-03-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd