fluent-plugin-elasticsearch 4.0.1 → 4.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d857451dc47373e32d9b2833fcb63a6afca5e45c1d131ee229ff8e29e408955
-  data.tar.gz: c0b7f154abe0b1e8288f72a7447c81a1c96ef3c7bb939af9f5964257e413a926
+  metadata.gz: 6c79a2fb7a4b83811b236661509fedd9c32b6287c085bbaac3f2bd758f4e127d
+  data.tar.gz: d68ae2e3f37593f3dc485de3ebbd6c5eb29c0f9e9d4bf3b10ec95a57ccd4a610
 SHA512:
-  metadata.gz: 1366c860429fe8da518a618f3651c839d5e151fbdf8025557433e323c84b718b7132fb27ad2c5c40d7d72980dcdb430a3398cb74aa1aafd99e4c4f701564c56d
-  data.tar.gz: caba8636b01815d08a00a81bf8159eaeafa0fce7824f2408d11f813f23504ab63b5ab1076c0a47ebc6f786f0228765a50f26b2076b5d1004710b1b448dace176
+  metadata.gz: cdd15e1d42d39d1b3ce014490201733472265efd7f220434a59bf1d910ec4fcd405ebdcc2c89c6bf9e5a19ac58204c9c3d974e4900c47fcbbb8337b8c7ec9321
+  data.tar.gz: cf13ac3ca840cc2e481485eb33b52e94ff9629aae7148d86d2213f3af65ad67771b7f6049d01d4027a377abb8acfde2c77c6d793613b583452333d72c1e98482

data/History.md

@@ -1,6 +1,9 @@
 ## Changelog [[tags]](https://github.com/uken/fluent-plugin-elasticsearch/tags)
 
 ### [Unreleased]
+### 4.0.2
+- Support TLSv1.3 (#710)
+
 ### 4.0.1
 - Placeholders for template name and customize template (#708)
 - Add overwriting ilm policy config parameter (#707)

data/README.md

@@ -838,6 +838,18 @@ ssl_version TLSv1_2 # or [SSLv23, TLSv1, TLSv1_1]
 
 :warning: If SSL/TLS enabled, it might have to be required to set ssl\_version.
 
+In Elasticsearch plugin v4.0.2 with Ruby 2.5 or later combination, Elasticsearch plugin also support `ssl_max_version` and `ssl_min_version`.
+
+```
+ssl_max_version TLSv1_3
+ssl_min_version TLSv1_2
+```
+
+Elasticsearch plugin will use TLSv1.2 as minimum ssl version and TLSv1.3 as maximum ssl version on transportation with TLS. Note that when they are used in Elastissearch plugin configuration, *`ssl_version` is not used* to set up TLS version.
+
+If they are *not* specified in the Elasticsearch plugin configuration, the value of `ssl_version` will be *used in `ssl_max_version` and `ssl_min_version`*.
+
+
 ### Proxy Support
 
 Starting with version 0.8.0, this gem uses excon, which supports proxy with environment variables - https://github.com/excon/excon#proxy-support
@@ -1231,6 +1243,13 @@ If you want to use TLS v1.2, please use `ssl_version` parameter like as:
 ssl_version TLSv1_2
 ```
 
+or, in v4.0.2 or later with Ruby 2.5 or later combination, the following congiuration is also valid:
+
+```
+ssl_max_version TLSv1_2
+ssl_min_version TLSv1_2
+```
+
 ### Cannot connect TLS enabled reverse Proxy
 
 A common cause of failure is that you are trying to connect to an Elasticsearch instance behind nginx reverse proxy which uses an incompatible ssl protocol version.
@@ -1322,6 +1341,13 @@ If you want to use TLS v1.2, please use `ssl_version` parameter like as:
 ssl_version TLSv1_2
 ```
 
+or, in v4.0.2 or later with Ruby 2.5 or later combination, the following congiuration is also valid:
+
+```
+ssl_max_version TLSv1_2
+ssl_min_version TLSv1_2
+```
+
 ### Declined logs are resubmitted forever, why?
 
 Sometimes users write Fluentd configuration like this:

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '4.0.1'
+  s.version       = '4.0.2'
   s.authors       = ['diogo', 'pitr', 'Hiroshi Hatake']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/elasticsearch_tls.rb

@@ -0,0 +1,70 @@
+require 'openssl'
+require 'fluent/configurable'
+require 'fluent/config/error'
+
+module Fluent::Plugin
+  module ElasticsearchTLS
+    SUPPORTED_TLS_VERSIONS = if defined?(OpenSSL::SSL::TLS1_3_VERSION)
+                               [:TLSv1, :TLSv1_1, :TLSv1_2, :TLSv1_3].freeze
+                             else
+                               [:SSLv23, :TLSv1, :TLSv1_1, :TLSv1_2].freeze
+                             end
+
+    DEFAULT_VERSION = :TLSv1
+    METHODS_MAP = begin
+                    # When openssl supports OpenSSL::SSL::TLSXXX constants representations, we use them.
+                    map = {
+                      TLSv1: OpenSSL::SSL::TLS1_VERSION,
+                      TLSv1_1: OpenSSL::SSL::TLS1_1_VERSION,
+                      TLSv1_2: OpenSSL::SSL::TLS1_2_VERSION
+                    }
+                    map[:TLSv1_3] = OpenSSL::SSL::TLS1_3_VERSION if defined?(OpenSSL::SSL::TLS1_3_VERSION)
+                    USE_TLS_MINMAX_VERSION = true
+                    map.freeze
+                  rescue NameError
+                    map = {
+                      SSLv23: :SSLv23,
+                      TLSv1: :TLSv1,
+                      TLSv1_1: :TLSv1_1,
+                      TLSv1_2: :TLSv1_2,
+                    }
+                    USE_TLS_MINMAX_VERSION = false
+                  end
+    private_constant :METHODS_MAP
+
+    module ElasticsearchTLSParams
+      include Fluent::Configurable
+
+      config_param :ssl_version, :enum, list: Fluent::Plugin::ElasticsearchTLS::SUPPORTED_TLS_VERSIONS, default: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION
+      config_param :ssl_min_version, :enum, list: Fluent::Plugin::ElasticsearchTLS::SUPPORTED_TLS_VERSIONS, default: nil
+      config_param :ssl_max_version, :enum, list: Fluent::Plugin::ElasticsearchTLS::SUPPORTED_TLS_VERSIONS, default: nil
+    end
+
+    def self.included(mod)
+      mod.include ElasticsearchTLSParams
+    end
+
+    def set_tls_minmax_version_config(ssl_version, ssl_max_version, ssl_min_version)
+      if USE_TLS_MINMAX_VERSION
+        case
+        when ssl_min_version.nil? && ssl_max_version.nil?
+          ssl_min_version = METHODS_MAP[ssl_version]
+          ssl_max_version = METHODS_MAP[ssl_version]
+        when ssl_min_version && ssl_max_version.nil?
+          raise Fluent::ConfigError, "When you set 'ssl_min_version', must set 'ssl_max_version' together."
+        when ssl_min_version.nil? && ssl_max_version
+          raise Fluent::ConfigError, "When you set 'ssl_max_version', must set 'ssl_min_version' together."
+        else
+          ssl_min_version = METHODS_MAP[ssl_min_version]
+          ssl_max_version = METHODS_MAP[ssl_max_version]
+        end
+
+        {max_version: ssl_max_version, min_version: ssl_min_version}
+      else
+        log.warn "'ssl_min_version' does not have any effect in this environment. Use 'ssl_version' instead." unless ssl_min_version.nil?
+        log.warn "'ssl_max_version' does not have any effect in this environment. Use 'ssl_version' instead." unless ssl_max_version.nil?
+        {version: ssl_version}
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -24,6 +24,7 @@ require_relative 'elasticsearch_error'
 require_relative 'elasticsearch_error_handler'
 require_relative 'elasticsearch_index_template'
 require_relative 'elasticsearch_index_lifecycle_management'
+require_relative 'elasticsearch_tls'
 begin
   require_relative 'oj_serializer'
 rescue LoadError
@@ -53,6 +54,7 @@ module Fluent::Plugin
 
     attr_reader :alias_indexes
     attr_reader :template_names
+    attr_reader :ssl_version_options
 
     helpers :event_emitter, :compat_parameters, :record_accessor
 
@@ -104,7 +106,6 @@ EOC
     config_param :client_cert, :string, :default => nil
     config_param :client_key_pass, :string, :default => nil, :secret => true
     config_param :ca_file, :string, :default => nil
-    config_param :ssl_version, :enum, list: [:SSLv23, :TLSv1, :TLSv1_1, :TLSv1_2], :default => :TLSv1
     config_param :remove_keys, :string, :default => nil
     config_param :remove_keys_on_update, :string, :default => ""
     config_param :remove_keys_on_update_key, :string, :default => nil
@@ -167,6 +168,7 @@ EOC
     include Fluent::ElasticsearchIndexTemplate
     include Fluent::Plugin::ElasticsearchConstants
     include Fluent::Plugin::ElasticsearchIndexLifecycleManagement
+    include Fluent::Plugin::ElasticsearchTLS
 
     def initialize
       super
@@ -184,6 +186,7 @@ EOC
       end
       @time_parser = create_time_parser
       @backend_options = backend_options
+      @ssl_version_options = set_tls_minmax_version_config(@ssl_version, @ssl_max_version, @ssl_min_version)
 
       if @remove_keys
         @remove_keys = @remove_keys.split(/\s*,\s*/)
@@ -501,6 +504,7 @@ EOC
                          {}
                        end
         headers = { 'Content-Type' => @content_type.to_s }.merge(@custom_headers).merge(gzip_headers)
+        ssl_options = { verify: @ssl_verify, ca_file: @ca_file}.merge(@ssl_version_options)
 
         transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
                                                                             options: {
@@ -511,7 +515,7 @@ EOC
                                                                               transport_options: {
                                                                                 headers: headers,
                                                                                 request: { timeout: @request_timeout },
-                                                                                ssl: { verify: @ssl_verify, ca_file: @ca_file, version: @ssl_version }
+                                                                                ssl: ssl_options,
                                                                               },
                                                                               http: {
                                                                                 user: @user,

data/lib/fluent/plugin/out_elasticsearch_dynamic.rb

@@ -50,6 +50,7 @@ module Fluent::Plugin
                          {}
                        end
         headers = { 'Content-Type' => @content_type.to_s, }.merge(gzip_headers)
+        ssl_options = { verify: @ssl_verify, ca_file: @ca_file}.merge(@ssl_version_options)
         transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
                                                                             options: {
                                                                               reload_connections: @reload_connections,
@@ -59,7 +60,7 @@ module Fluent::Plugin
                                                                               transport_options: {
                                                                                 headers: headers,
                                                                                 request: { timeout: @request_timeout },
-                                                                                ssl: { verify: @ssl_verify, ca_file: @ca_file, version: @ssl_version }
+                                                                                ssl: ssl_options,
                                                                               },
                                                                               http: {
                                                                                 user: @user,

data/test/plugin/test_elasticsearch_tls.rb

@@ -0,0 +1,139 @@
+require_relative '../helper'
+require 'fluent/test/driver/output'
+require 'fluent/plugin/output'
+require 'fluent/plugin/elasticsearch_tls'
+
+class TestElasticsearchTLS < Test::Unit::TestCase
+
+  class TestTLSModuleOutput < Fluent::Plugin::Output
+    include Fluent::Plugin::ElasticsearchTLS
+
+    def initialize
+      super
+      @emit_streams = []
+    end
+
+    def write(chunk)
+      es = Fluent::ArrayEventStream.new
+      chunk.each do |time, record|
+        es.add(time, record)
+      end
+      @emit_streams << [tag, es]
+    end
+  end
+
+  setup do
+    Fluent::Test.setup
+    @use_tls_minmax_version = begin
+                                map = {
+                                  TLSv1: OpenSSL::SSL::TLS1_VERSION,
+                                  TLSv1_1: OpenSSL::SSL::TLS1_1_VERSION,
+                                  TLSv1_2: OpenSSL::SSL::TLS1_2_VERSION
+                                }
+                                map[:TLSv1_3] = OpenSSL::SSL::TLS1_3_VERSION if defined?(OpenSSL::SSL::TLS1_3_VERSION)
+                                true
+                              rescue NameError
+                                false
+                              end
+    @enabled_tlsv1_3 = begin
+                         map = {TLSv1_3: OpenSSL::SSL::TLS1_3_VERSION}
+                         true
+                       rescue NameError
+                         false
+                       end
+  end
+
+  def driver(conf='')
+    Fluent::Test::Driver::Output.new(TestTLSModuleOutput).configure(conf)
+  end
+
+  test 'configure' do
+    assert_equal Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION, driver.instance.ssl_version
+    assert_nil driver.instance.ssl_max_version
+    assert_nil driver.instance.ssl_min_version
+  end
+
+  test 'check USE_TLS_MINMAX_VERSION value' do
+    assert_equal @use_tls_minmax_version, Fluent::Plugin::ElasticsearchTLS::USE_TLS_MINMAX_VERSION
+  end
+
+  sub_test_case 'set_tls_minmax_version_config' do
+    test 'default' do
+      d = driver('')
+      ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version, nil, nil)
+      if @use_tls_minmax_version
+        assert_equal({max_version: OpenSSL::SSL::TLS1_VERSION,
+                      min_version: OpenSSL::SSL::TLS1_VERSION}, ssl_version_options)
+      else
+        assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION}, ssl_version_options)
+      end
+    end
+
+    test 'errorous cases' do
+      if @use_tls_minmax_version
+        assert_raise(Fluent::ConfigError) do
+          d = driver(%{ssl_max_version TLSv1_2})
+          d.instance.set_tls_minmax_version_config(d.instance.ssl_version,
+                                                   d.instance.ssl_max_version,
+                                                   d.instance.ssl_min_version)
+        end
+        assert_raise(Fluent::ConfigError) do
+          d = driver(%{ssl_min_version TLSv1_2})
+          d.instance.set_tls_minmax_version_config(d.instance.ssl_version,
+                                                   d.instance.ssl_max_version,
+                                                   d.instance.ssl_min_version)
+        end
+      else
+        d1 = driver(%{
+          ssl_max_version TLSv1_2
+          @log_level info
+        })
+        d1.instance.set_tls_minmax_version_config(d1.instance.ssl_version,
+                                                  d1.instance.ssl_max_version,
+                                                  d1.instance.ssl_min_version)
+
+        d1.logs.any? {|a| a.include?("'ssl_max_version' does not have any effect in this environment.") }
+        d2 = driver(%{
+          ssl_min_version TLSv1_2
+          @log_level info
+        })
+        d2.instance.set_tls_minmax_version_config(d2.instance.ssl_version,
+                                                  d2.instance.ssl_max_version,
+                                                  d2.instance.ssl_min_version)
+        d2.logs.any? {|a| a.include?("'ssl_min_version' does not have any effect in this environment.") }
+      end
+    end
+
+    test 'min_version & max_version' do
+      config = %{
+        ssl_max_version TLSv1_2
+        ssl_min_version TLSv1_1
+      }
+      d = driver(config)
+      ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version,
+                                                                     d.instance.ssl_max_version,
+                                                                     d.instance.ssl_min_version)
+      if @use_tls_minmax_version
+        assert_equal({max_version: OpenSSL::SSL::TLS1_2_VERSION,
+                      min_version: OpenSSL::SSL::TLS1_1_VERSION}, ssl_version_options)
+      else
+        assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION}, ssl_version_options)
+      end
+    end
+
+    test 'TLSv1.3' do
+      omit "openssl gem does not support TLSv1.3" unless @enabled_tlsv1_3
+      config = %{
+        ssl_max_version TLSv1_3
+        ssl_min_version TLSv1_2
+      }
+      d = driver(config)
+      ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version,
+                                                                     d.instance.ssl_max_version,
+                                                                     d.instance.ssl_min_version)
+      assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
+                    min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
+
+    end
+  end
+end

data/test/plugin/test_out_elasticsearch.rb

@@ -218,7 +218,16 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal '/es/', instance.path
     assert_equal 'john', instance.user
     assert_equal 'doe', instance.password
-    assert_equal :TLSv1, instance.ssl_version
+    assert_equal Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION, instance.ssl_version
+    assert_nil instance.ssl_max_version
+    assert_nil instance.ssl_min_version
+    if Fluent::Plugin::ElasticsearchTLS::USE_TLS_MINMAX_VERSION
+      assert_equal({max_version: OpenSSL::SSL::TLS1_VERSION, min_version: OpenSSL::SSL::TLS1_VERSION},
+                   instance.ssl_version_options)
+    else
+      assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION},
+                   instance.ssl_version_options)
+    end
     assert_nil instance.client_key
     assert_nil instance.client_cert
     assert_nil instance.client_key_pass

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -97,7 +97,16 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     assert_equal 'john', instance.user
     assert_equal 'doe', instance.password
     assert_equal '/es/', instance.path
-    assert_equal :TLSv1, instance.ssl_version
+    assert_equal Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION, instance.ssl_version
+    assert_nil instance.ssl_max_version
+    assert_nil instance.ssl_min_version
+    if Fluent::Plugin::ElasticsearchTLS::USE_TLS_MINMAX_VERSION
+      assert_equal({max_version: OpenSSL::SSL::TLS1_VERSION, min_version: OpenSSL::SSL::TLS1_VERSION},
+                   instance.ssl_version_options)
+    else
+      assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION},
+                   instance.ssl_version_options)
+    end
     assert_nil instance.client_key
     assert_nil instance.client_cert
     assert_nil instance.client_key_pass

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.0.1
+  version: 4.0.2
 platform: ruby
 authors:
 - diogo
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-31 00:00:00.000000000 Z
+date: 2020-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -157,6 +157,7 @@ files:
 - lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb
 - lib/fluent/plugin/elasticsearch_index_template.rb
 - lib/fluent/plugin/elasticsearch_simple_sniffer.rb
+- lib/fluent/plugin/elasticsearch_tls.rb
 - lib/fluent/plugin/filter_elasticsearch_genid.rb
 - lib/fluent/plugin/in_elasticsearch.rb
 - lib/fluent/plugin/oj_serializer.rb
@@ -166,6 +167,7 @@ files:
 - test/plugin/test_alias_template.json
 - test/plugin/test_elasticsearch_error_handler.rb
 - test/plugin/test_elasticsearch_index_lifecycle_management.rb
+- test/plugin/test_elasticsearch_tls.rb
 - test/plugin/test_filter_elasticsearch_genid.rb
 - test/plugin/test_in_elasticsearch.rb
 - test/plugin/test_out_elasticsearch.rb
@@ -201,6 +203,7 @@ test_files:
 - test/plugin/test_alias_template.json
 - test/plugin/test_elasticsearch_error_handler.rb
 - test/plugin/test_elasticsearch_index_lifecycle_management.rb
+- test/plugin/test_elasticsearch_tls.rb
 - test/plugin/test_filter_elasticsearch_genid.rb
 - test/plugin/test_in_elasticsearch.rb
 - test/plugin/test_out_elasticsearch.rb