fluent-plugin-elasticsearch 4.0.11 → 4.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c90e0b347a5c90ec23231df927384dadcc4b91b813a4e2c11df23539dfe05628
-  data.tar.gz: d6e296c8006870a5f85bd0b97294c69d797fc3f40b574ede7645fcd808ae6356
+  metadata.gz: 12f88bef6f61ff3acbbd654c63a6402f7374678e78cf438f2026551246e9e0fc
+  data.tar.gz: 7c8ffbd9626bf8204820326ed7a3bb9157f0e3e6b406dbba51f2cb66c899b6ca
 SHA512:
-  metadata.gz: 8466c4e3a13ef5c5ca1bda536a0baf7044fdaa0bc3ae1e1c7e6d0e6cf3dc7596d2b68e3f61ed2fb788f8ef2d8392939b8f63fd6aedc53e5818725bc6e0910aa6
-  data.tar.gz: b0572e50952ce49e44b280e94764d0d204d482e8e89cc2124e9a3025c1bd4cf7debe5625ad141337e5f81e6b6816286ee81f931f6b782f0a8cffcbfa0ed12dc3
+  metadata.gz: 1bc665321d8aed75d579a638d11d9254f2f3b50a4b2ce16b03e2db0ad2742fce155352c792cdeed1b244945430598d170eae546ab1e0cd4d4beedf5f0d9ea2c2
+  data.tar.gz: 6f54a08d03c458e7cf3dced5df532fbb5f5e2df2c9e8669c52a9bb3ebb0da231246717384653add9e585cbd2a068c00ab5be0fd12505fd04fccb7d0ca970faa3

data/History.md

@@ -2,6 +2,9 @@
 
 ### [Unreleased]
 
+### 4.1.0
+- Implement Fallback selector and configurable selector class (#782)
+
 ### 4.0.11
 - Add custom and time placeholders combination testcase for ILM (#781)
 - Really support ILM on `logstash_format` enabled environment (#779)

data/README.md

@@ -78,6 +78,7 @@ Current maintainers: @cosmo0920
   + [Hash flattening](#hash-flattening)
   + [Generate Hash ID](#generate-hash-id)
   + [sniffer_class_name](#sniffer-class-name)
+  + [selector_class_name](#selector-class-name)
   + [reload_after](#reload-after)
   + [validate_client_version](#validate-client-version)
   + [unrecoverable_error_types](#unrecoverable-error-types)
@@ -987,6 +988,18 @@ sniffer_class_name Fluent::Plugin::ElasticsearchSimpleSniffer
 reload_after 100
 ```
 
+### Selector Class Name
+
+The default selector used by the `Elasticsearch::Transport` class works well when Fluentd should round robin and random selector cases. This doesn't work well when Fluentd should fallback behavior.
+The parameter `selector_class_name` gives you the ability to provide your own Selector class to implement whatever selection nodes logic you require.
+
+The below configuration is using plugin built-in `ElasticseatchFallbackSelector`:
+
+```
+hosts exhausted-host:9201,normal-host:9200
+selector_class_name "Fluent::Plugin::ElasticseatchFallbackSelector"
+```
+
 #### Tips
 
 The included sniffer class does not required `out_elasticsearch`.

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '4.0.11'
+  s.version       = '4.1.0'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_fallback_selector.rb

@@ -0,0 +1,9 @@
+require 'elasticsearch/transport/transport/connections/selector'
+
+class Fluent::Plugin::ElasticseatchFallbackSelector
+  include Elasticsearch::Transport::Transport::Connections::Selector::Base
+
+  def select(options={})
+    connections.first
+  end
+end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -25,6 +25,7 @@ require_relative 'elasticsearch_error_handler'
 require_relative 'elasticsearch_index_template'
 require_relative 'elasticsearch_index_lifecycle_management'
 require_relative 'elasticsearch_tls'
+require_relative 'elasticsearch_fallback_selector'
 begin
   require_relative 'oj_serializer'
 rescue LoadError
@@ -137,6 +138,7 @@ EOC
     config_param :with_transporter_log, :bool, :default => false
     config_param :emit_error_for_missing_id, :bool, :default => false
     config_param :sniffer_class_name, :string, :default => nil
+    config_param :selector_class_name, :string, :default => nil
     config_param :reload_after, :integer, :default => DEFAULT_RELOAD_AFTER
     config_param :content_type, :enum, list: [:"application/json", :"application/x-ndjson"], :default => :"application/json",
                  :deprecated => <<EOC
@@ -300,6 +302,13 @@ EOC
         raise Fluent::ConfigError, "Could not load sniffer class #{@sniffer_class_name}: #{ex}"
       end
 
+      @selector_class = nil
+      begin
+        @selector_class = Object.const_get(@selector_class_name) if @selector_class_name
+      rescue Exception => ex
+        raise Fluent::ConfigError, "Could not load selector class #{@selector_class_name}: #{ex}"
+      end
+
       @last_seen_major_version = if major_version = handle_last_seen_es_major_version
                                    major_version
                                  else
@@ -566,6 +575,7 @@ EOC
                                                                               },
                                                                               sniffer_class: @sniffer_class,
                                                                               serializer_class: @serializer_class,
+                                                                              selector_class: @selector_class,
                                                                               compression: compress_connection,
                                                                             }), &adapter_conf)
         Elasticsearch::Client.new transport: transport

data/test/plugin/test_elasticsearch_fallback_selector.rb

@@ -0,0 +1,73 @@
+require_relative '../helper'
+require 'fluent/test/driver/output'
+require 'fluent/plugin/out_elasticsearch'
+
+class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
+  attr_accessor :index_cmds
+
+  def setup
+    Fluent::Test.setup
+    @driver = nil
+    log = Fluent::Engine.log
+    log.out.logs.slice!(0, log.out.logs.length)
+  end
+
+  def stub_elastic(url="http://localhost:9200/_bulk")
+    stub_request(:post, url).with do |req|
+      @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
+    end
+  end
+
+  def stub_elastic_info(url="http://localhost:9200/", version="6.4.2")
+    body ="{\"version\":{\"number\":\"#{version}\"}}"
+    stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
+  end
+
+  def stub_elastic_info_not_found(url="http://localhost:9200/", version="6.4.2")
+    stub_request(:get, url).to_return(:status => [404, "Not Found"])
+  end
+
+  def stub_elastic_info_unavailable(url="http://localhost:9200/", version="6.4.2")
+    stub_request(:get, url).to_return(:status => [503, "Service Unavailable"])
+  end
+
+  def sample_record(content={})
+    {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}.merge(content)
+  end
+
+  def driver(conf='')
+    @driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput) {
+      # v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
+      if !defined?(Fluent::Plugin::Output)
+        def format(tag, time, record)
+          [time, record].to_msgpack
+        end
+      end
+    }.configure(conf)
+  end
+
+  def test_fallback_on_info
+    stub_elastic_info_not_found("http://localhost:9202/")
+    stub_elastic_info_unavailable("http://localhost:9201/")
+    stub_elastic_info
+    stub_elastic
+    config = %[
+      hosts localhost:9202,localhost:9201,localhost:9200
+      selector_class_name Fluent::Plugin::ElasticseatchFallbackSelector
+      @log_level debug
+      with_transporter_log true
+      reload_connections true
+      reload_after 10
+    ]
+    assert_raise(Elasticsearch::Transport::Transport::Errors::NotFound) do
+      driver(config)
+    end
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record)
+    end
+    assert_equal(2, index_cmds.length)
+    assert_equal("fluentd", index_cmds.first['index']['_index'])
+  end
+
+  # TODO: on feed phase test case
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.0.11
+  version: 4.1.0
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-09 00:00:00.000000000 Z
+date: 2020-07-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -160,6 +160,7 @@ files:
 - lib/fluent/plugin/elasticsearch_constants.rb
 - lib/fluent/plugin/elasticsearch_error.rb
 - lib/fluent/plugin/elasticsearch_error_handler.rb
+- lib/fluent/plugin/elasticsearch_fallback_selector.rb
 - lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb
 - lib/fluent/plugin/elasticsearch_index_template.rb
 - lib/fluent/plugin/elasticsearch_simple_sniffer.rb
@@ -172,6 +173,7 @@ files:
 - test/helper.rb
 - test/plugin/test_alias_template.json
 - test/plugin/test_elasticsearch_error_handler.rb
+- test/plugin/test_elasticsearch_fallback_selector.rb
 - test/plugin/test_elasticsearch_index_lifecycle_management.rb
 - test/plugin/test_elasticsearch_tls.rb
 - test/plugin/test_filter_elasticsearch_genid.rb
@@ -209,6 +211,7 @@ test_files:
 - test/helper.rb
 - test/plugin/test_alias_template.json
 - test/plugin/test_elasticsearch_error_handler.rb
+- test/plugin/test_elasticsearch_fallback_selector.rb
 - test/plugin/test_elasticsearch_index_lifecycle_management.rb
 - test/plugin/test_elasticsearch_tls.rb
 - test/plugin/test_filter_elasticsearch_genid.rb