fluent-plugin-elasticsearch 2.11.4 → 2.11.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ee8aaeedb5eda1f4857a091853a1c479e7d455d0e44db13eea8591aa93424ee0
-  data.tar.gz: 96dd085e19d9ee4e22d5f3844a366c7fe1e737a5426240259911295758d735ad
+  metadata.gz: 36408493fd7023fcd54d9c3e1ad98f3fa897d7f43dcb0a2d8ebc2e789bbd9518
+  data.tar.gz: 1cf82341e33dc2b713a94181b9450bbdaaf82f02c025b69b7008eb4e45a7314f
 SHA512:
-  metadata.gz: b460e08f99c4c663e432ffa8adafb065b005680d5a534b67ae9485ae01552bc14c0f6fe11b9c0afc714987871977c8e7a751ab980b54df63704d8238405d282f
-  data.tar.gz: e6a83a30052a48a8c6986fd47fec45b1e6c700d0b2a9a39d7d8379d897907df08a648ec348cc445d91f2e2ebd399506a3aad6a4553b91a45ff110b11f1680992
+  metadata.gz: 9cfe34fbe403a4ef5233a1e34a42e02e184228a22bf20a79698912176a1bfb4ab03f21fd2be0015a666ced2be46106218013e6946bd5b5351b30d08993c08aca
+  data.tar.gz: 9712390a2f6be181cf642583d01af895469183a5198513c40493fa61982c9e05198a670bbe74cb4e79e545f145c0330e45f463c1f928572b5421703d19344ff8

data/History.md

@@ -2,6 +2,9 @@
 
 ### [Unreleased]
 
+### 2.11.5
+- Uplift Merge pull request #459 from richm/v0.12-simple-sniffer (#461)
+
 ### 2.11.4
 - Persistent backend (#456)
 

data/README.md

@@ -62,6 +62,8 @@ Current maintainers: @cosmo0920
   + [Buffer options](#buffer-options)
   + [Hash flattening](#hash-flattening)
   + [Generate Hash ID](#generate-hash-id)
+  + [sniffer_class_name](#sniffer_class_name)
+  + [reload_after](#reload_after)
   + [Not seeing a config you need?](#not-seeing-a-config-you-need)
   + [Dynamic configuration](#dynamic-configuration)
   + [Placeholders](#placeholders)
@@ -716,6 +718,29 @@ Here is a sample config:
 </match>
 ```
 
+### Sniffer Class Name
+
+The default Sniffer used by the `Elasticsearch::Transport` class works well when Fluentd has a direct connection
+to all of the Elasticsearch servers and can make effective use of the `_nodes` API.  This doesn't work well
+when Fluentd must connect through a load balancer or proxy.  The parameter `sniffer_class_name` gives you the
+ability to provide your own Sniffer class to implement whatever connection reload logic you require.  In addition,
+there is a new `Fluent::Plugin::ElasticsearchSimpleSniffer` class which reuses the hosts given in the configuration, which
+is typically the hostname of the load balancer or proxy.  For example, a configuration like this would cause
+connections to `logging-es` to reload every 100 operations:
+
+```
+host logging-es
+port 9200
+reload_connections true
+sniffer_class_name Fluent::Plugin::ElasticsearchSimpleSniffer
+reload_after 100
+```
+
+### Reload After
+
+When `reload_connections true`, this is the integer number of operations after which the plugin will
+reload the connections.  The default value is 10000.
+
 ### Not seeing a config you need?
 
 We try to keep the scope of this plugin small and not add too many configuration options. If you think an option would be useful to others, feel free to open an issue or contribute a Pull Request.

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '2.11.4'
+  s.version       = '2.11.5'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_simple_sniffer.rb

@@ -0,0 +1,10 @@
+require 'elasticsearch'
+
+class Fluent::Plugin::ElasticsearchSimpleSniffer < Elasticsearch::Transport::Transport::Sniffer
+
+  def hosts
+    @transport.logger.debug "In Fluent::Plugin::ElasticsearchSimpleSniffer hosts #{@transport.hosts}" if @transport.logger
+    @transport.hosts
+  end
+
+end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -44,6 +44,7 @@ module Fluent::Plugin
     DEFAULT_ELASTICSEARCH_VERSION = 5 # For compatibility.
     DEFAULT_TYPE_NAME_ES_7x = "_doc".freeze
     DEFAULT_TYPE_NAME = "fluentd".freeze
+    DEFAULT_RELOAD_AFTER = -1
 
     config_param :host, :string,  :default => 'localhost'
     config_param :port, :integer, :default => 9200
@@ -101,6 +102,8 @@ EOC
     config_param :pipeline, :string, :default => nil
     config_param :with_transporter_log, :bool, :default => false
     config_param :emit_error_for_missing_id, :bool, :default => false
+    config_param :sniffer_class_name, :string, :default => nil
+    config_param :reload_after, :integer, :default => DEFAULT_RELOAD_AFTER
     config_param :content_type, :enum, list: [:"application/json", :"application/x-ndjson"], :default => :"application/json",
                  :deprecated => <<EOC
 elasticsearch gem v6.0.2 starts to use correct Content-Type. Please upgrade elasticserach gem and stop to use this option.
@@ -207,6 +210,12 @@ EOC
           end
         end
       end
+      @sniffer_class = nil
+      begin
+        @sniffer_class = Object.const_get(@sniffer_class_name) if @sniffer_class_name
+      rescue Exception => ex
+        raise Fluent::ConfigError, "Could not load sniffer class #{@sniffer_class_name}: #{ex}"
+      end
     end
 
     def backend_options
@@ -272,9 +281,13 @@ EOC
     def client
       @_es ||= begin
         adapter_conf = lambda {|f| f.adapter @http_backend, @backend_options }
+        local_reload_connections = @reload_connections
+        if local_reload_connections && @reload_after > DEFAULT_RELOAD_AFTER
+          local_reload_connections = @reload_after
+        end
         transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(get_connection_options.merge(
                                                                             options: {
-                                                                              reload_connections: @reload_connections,
+                                                                              reload_connections: local_reload_connections,
                                                                               reload_on_failure: @reload_on_failure,
                                                                               resurrect_after: @resurrect_after,
                                                                               retry_on_failure: 5,
@@ -287,7 +300,8 @@ EOC
                                                                               http: {
                                                                                 user: @user,
                                                                                 password: @password
-                                                                              }
+                                                                              },
+                                                                              sniffer_class: @sniffer_class,
                                                                             }), &adapter_conf)
         es = Elasticsearch::Client.new transport: transport
 

data/test/plugin/test_out_elasticsearch.rb

@@ -2189,4 +2189,21 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(index_cmds.first['index']['_index'], nil)
   end
 
+  def test_use_simple_sniffer
+    require 'fluent/plugin/elasticsearch_simple_sniffer'
+    driver.configure("sniffer_class_name Fluent::Plugin::ElasticsearchSimpleSniffer
+                      log_level debug
+                      with_transporter_log true
+                      reload_connections true
+                      reload_after 1")
+    stub_elastic_ping
+    stub_elastic
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+    log = driver.logs
+    # 2 - one for the ping, one for the _bulk
+    assert_logs_include(log, /In Fluent::Plugin::ElasticsearchSimpleSniffer hosts/, 2)
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.11.4
+  version: 2.11.5
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-08 00:00:00.000000000 Z
+date: 2018-08-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -147,6 +147,7 @@ files:
 - lib/fluent/plugin/elasticsearch_constants.rb
 - lib/fluent/plugin/elasticsearch_error_handler.rb
 - lib/fluent/plugin/elasticsearch_index_template.rb
+- lib/fluent/plugin/elasticsearch_simple_sniffer.rb
 - lib/fluent/plugin/filter_elasticsearch_genid.rb
 - lib/fluent/plugin/out_elasticsearch.rb
 - lib/fluent/plugin/out_elasticsearch_dynamic.rb