fluent-plugin-elasticsearch 1.17.1 → 1.17.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b520bf633ecbf621fb94e7f4bfa83fa1922d678a577ab343b5b4936d602ceb4e
-  data.tar.gz: 9b05cb7ea5baffe76509387d61cb50c6ac1dffc097e273148ffae9b719f7cdf5
+  metadata.gz: 331ba0c76606e47fa96214c432b7c67bdeda5440cb45c8689f9b63052f372ec1
+  data.tar.gz: 68a57f20ce85df802931dc206c52874acb8bf8d49d5d30c020f43a295a8ffe26
 SHA512:
-  metadata.gz: 3cf3aca2d2ed49e9807c17a7c740a001eb627575406f9336c69bdab6fe8a2f6156873a2488fda9947abc1c16332700497bcd2ebfe32eda53352acbdc1feb3b5e
-  data.tar.gz: b243859f0956ef22a561d1755a3a73700fb1cc82a18b857aa0248d068e72e40afd3ca78eeb3356fc898ccc6d5271504073cedae7aaa73a0b478eb4c89bb1a4f3
+  metadata.gz: 45d64db8c27ba51856ad811ee09c509ee412bd715abfc7d63a1d0314e07d42295256e8b667df8f81ca2c48425cdc87b1a3808712493c7eb27bb0ab05d2706b3c
+  data.tar.gz: 473bc12ab5f0d3e0748f5ea0a1a419ca584a49b76df603e472411559892bf5c8d7beef25e25913270ad6dba1ac7435982955747f1bc53765c5dd6583e663bf3a

data/History.md

@@ -2,6 +2,9 @@
 
 ### [Unreleased]
 
+### 1.17.2
+- add simple sniffer for simple proxy/lb cases (#459)
+
 ### 1.17.1
 - backport strictness-scheme (#447)
 

data/README.md

@@ -56,6 +56,8 @@ Note: For Amazon Elasticsearch Service please consider using [fluent-plugin-aws-
   + [Buffered output options](#buffered-output-options)
   + [Hash flattening](#hash-flattening)
   + [Generate Hash ID](#generate-hash-id)
+  + [sniffer_class_name](#sniffer_class_name)
+  + [reload_after](#reload_after)
   + [Not seeing a config you need?](#not-seeing-a-config-you-need)
   + [Dynamic configuration](#dynamic-configuration)
 * [Contact](#contact)
@@ -576,6 +578,29 @@ Here is a sample config:
 </match>
 ```
 
+### Sniffer Class Name
+
+The default Sniffer used by the `Elasticsearch::Transport` class works well when Fluentd has a direct connection
+to all of the Elasticsearch servers and can make effective use of the `_nodes` API.  This doesn't work well
+when Fluentd must connect through a load balancer or proxy.  The parameter `sniffer_class_name` gives you the
+ability to provide your own Sniffer class to implement whatever connection reload logic you require.  In addition,
+there is a new `Fluent::ElasticsearchSimpleSniffer` class which reuses the hosts given in the configuration, which
+is typically the hostname of the load balancer or proxy.  For example, a configuration like this would cause
+connections to `logging-es` to reload every 100 operations:
+
+```
+host logging-es
+port 9200
+reload_connections true
+sniffer_class_name Fluent::ElasticsearchSimpleSniffer
+reload_after 100
+```
+
+### Reload After
+
+When `reload_connections true`, this is the integer number of operations after which the plugin will
+reload the connections.  The default value is 10000.
+
 ### Not seeing a config you need?
 
 We try to keep the scope of this plugin small and not add too many configuration options. If you think an option would be useful to others, feel free to open an issue or contribute a Pull Request.

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '1.17.1'
+  s.version       = '1.17.2'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_simple_sniffer.rb

@@ -0,0 +1,10 @@
+require 'elasticsearch'
+
+class Fluent::ElasticsearchSimpleSniffer < Elasticsearch::Transport::Transport::Sniffer
+
+  def hosts
+    @transport.logger.debug "In Fluent::ElasticsearchSimpleSniffer hosts #{@transport.hosts}" if @transport.logger
+    @transport.hosts
+  end
+
+end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -34,6 +34,8 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
 
   Fluent::Plugin.register_output('elasticsearch', self)
 
+  DEFAULT_RELOAD_AFTER = -1
+
   config_param :host, :string,  :default => 'localhost'
   config_param :port, :integer, :default => 9200
   config_param :user, :string, :default => nil
@@ -86,6 +88,8 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
   config_param :pipeline, :string, :default => nil
   config_param :with_transporter_log, :bool, :default => false
   config_param :emit_error_for_missing_id, :bool, :default => false
+  config_param :sniffer_class_name, :string, :default => nil
+  config_param :reload_after, :integer, :default => DEFAULT_RELOAD_AFTER
 
   include Fluent::ElasticsearchIndexTemplate
   include Fluent::ElasticsearchConstants
@@ -147,6 +151,13 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
       log.warn "Consider to specify log_level with @log_level." unless log_level
     end
 
+    @sniffer_class = nil
+    begin
+      @sniffer_class = Object.const_get(@sniffer_class_name) if @sniffer_class_name
+    rescue Exception => ex
+      raise Fluent::ConfigError, "Could not load sniffer class #{@sniffer_class_name}: #{ex}"
+    end
+
   end
 
   def create_meta_config_map
@@ -189,9 +200,13 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
     @_es ||= begin
       excon_options = { client_key: @client_key, client_cert: @client_cert, client_key_pass: @client_key_pass }
       adapter_conf = lambda {|f| f.adapter :excon, excon_options }
+      local_reload_connections = @reload_connections
+      if local_reload_connections && @reload_after > DEFAULT_RELOAD_AFTER
+        local_reload_connections = @reload_after
+      end
       transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(get_connection_options.merge(
                                                                           options: {
-                                                                            reload_connections: @reload_connections,
+                                                                            reload_connections: local_reload_connections,
                                                                             reload_on_failure: @reload_on_failure,
                                                                             resurrect_after: @resurrect_after,
                                                                             retry_on_failure: 5,
@@ -204,7 +219,8 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
                                                                             http: {
                                                                               user: @user,
                                                                               password: @password
-                                                                            }
+                                                                            },
+                                                                            sniffer_class: @sniffer_class,
                                                                           }), &adapter_conf)
       es = Elasticsearch::Client.new transport: transport
 

data/test/plugin/test_out_elasticsearch.rb

@@ -1716,4 +1716,20 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert(index_cmds[0].has_key?("create"))
   end
 
+  def test_use_simple_sniffer
+    require 'fluent/plugin/elasticsearch_simple_sniffer'
+    driver.configure("sniffer_class_name Fluent::ElasticsearchSimpleSniffer
+                      log_level debug
+                      with_transporter_log true
+                      reload_connections true
+                      reload_after 1")
+    stub_elastic_ping
+    stub_elastic
+    driver.emit(sample_record)
+    driver.run
+    log = driver.instance.router.emit_error_handler.log
+    # 2 - one for the ping, one for the _bulk
+    assert_logs_include(log.out.logs, /In Fluent::ElasticsearchSimpleSniffer hosts/, 2)
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.17.1
+  version: 1.17.2
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-07-20 00:00:00.000000000 Z
+date: 2018-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -147,6 +147,7 @@ files:
 - lib/fluent/plugin/elasticsearch_constants.rb
 - lib/fluent/plugin/elasticsearch_error_handler.rb
 - lib/fluent/plugin/elasticsearch_index_template.rb
+- lib/fluent/plugin/elasticsearch_simple_sniffer.rb
 - lib/fluent/plugin/filter_elasticsearch_genid.rb
 - lib/fluent/plugin/out_elasticsearch.rb
 - lib/fluent/plugin/out_elasticsearch_dynamic.rb