fluent-plugin-elasticsearch 4.0.1 → 4.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 2d857451dc47373e32d9b2833fcb63a6afca5e45c1d131ee229ff8e29e408955
4
- data.tar.gz: c0b7f154abe0b1e8288f72a7447c81a1c96ef3c7bb939af9f5964257e413a926
3
+ metadata.gz: 6c79a2fb7a4b83811b236661509fedd9c32b6287c085bbaac3f2bd758f4e127d
4
+ data.tar.gz: d68ae2e3f37593f3dc485de3ebbd6c5eb29c0f9e9d4bf3b10ec95a57ccd4a610
5
5
  SHA512:
6
- metadata.gz: 1366c860429fe8da518a618f3651c839d5e151fbdf8025557433e323c84b718b7132fb27ad2c5c40d7d72980dcdb430a3398cb74aa1aafd99e4c4f701564c56d
7
- data.tar.gz: caba8636b01815d08a00a81bf8159eaeafa0fce7824f2408d11f813f23504ab63b5ab1076c0a47ebc6f786f0228765a50f26b2076b5d1004710b1b448dace176
6
+ metadata.gz: cdd15e1d42d39d1b3ce014490201733472265efd7f220434a59bf1d910ec4fcd405ebdcc2c89c6bf9e5a19ac58204c9c3d974e4900c47fcbbb8337b8c7ec9321
7
+ data.tar.gz: cf13ac3ca840cc2e481485eb33b52e94ff9629aae7148d86d2213f3af65ad67771b7f6049d01d4027a377abb8acfde2c77c6d793613b583452333d72c1e98482
data/History.md CHANGED
@@ -1,6 +1,9 @@
1
1
  ## Changelog [[tags]](https://github.com/uken/fluent-plugin-elasticsearch/tags)
2
2
 
3
3
  ### [Unreleased]
4
+ ### 4.0.2
5
+ - Support TLSv1.3 (#710)
6
+
4
7
  ### 4.0.1
5
8
  - Placeholders for template name and customize template (#708)
6
9
  - Add overwriting ilm policy config parameter (#707)
data/README.md CHANGED
@@ -838,6 +838,18 @@ ssl_version TLSv1_2 # or [SSLv23, TLSv1, TLSv1_1]
838
838
 
839
839
  :warning: If SSL/TLS enabled, it might have to be required to set ssl\_version.
840
840
 
841
+ In Elasticsearch plugin v4.0.2 with Ruby 2.5 or later combination, Elasticsearch plugin also support `ssl_max_version` and `ssl_min_version`.
842
+
843
+ ```
844
+ ssl_max_version TLSv1_3
845
+ ssl_min_version TLSv1_2
846
+ ```
847
+
848
+ Elasticsearch plugin will use TLSv1.2 as minimum ssl version and TLSv1.3 as maximum ssl version on transportation with TLS. Note that when they are used in Elastissearch plugin configuration, *`ssl_version` is not used* to set up TLS version.
849
+
850
+ If they are *not* specified in the Elasticsearch plugin configuration, the value of `ssl_version` will be *used in `ssl_max_version` and `ssl_min_version`*.
851
+
852
+
841
853
  ### Proxy Support
842
854
 
843
855
  Starting with version 0.8.0, this gem uses excon, which supports proxy with environment variables - https://github.com/excon/excon#proxy-support
@@ -1231,6 +1243,13 @@ If you want to use TLS v1.2, please use `ssl_version` parameter like as:
1231
1243
  ssl_version TLSv1_2
1232
1244
  ```
1233
1245
 
1246
+ or, in v4.0.2 or later with Ruby 2.5 or later combination, the following congiuration is also valid:
1247
+
1248
+ ```
1249
+ ssl_max_version TLSv1_2
1250
+ ssl_min_version TLSv1_2
1251
+ ```
1252
+
1234
1253
  ### Cannot connect TLS enabled reverse Proxy
1235
1254
 
1236
1255
  A common cause of failure is that you are trying to connect to an Elasticsearch instance behind nginx reverse proxy which uses an incompatible ssl protocol version.
@@ -1322,6 +1341,13 @@ If you want to use TLS v1.2, please use `ssl_version` parameter like as:
1322
1341
  ssl_version TLSv1_2
1323
1342
  ```
1324
1343
 
1344
+ or, in v4.0.2 or later with Ruby 2.5 or later combination, the following congiuration is also valid:
1345
+
1346
+ ```
1347
+ ssl_max_version TLSv1_2
1348
+ ssl_min_version TLSv1_2
1349
+ ```
1350
+
1325
1351
  ### Declined logs are resubmitted forever, why?
1326
1352
 
1327
1353
  Sometimes users write Fluentd configuration like this:
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = 'fluent-plugin-elasticsearch'
6
- s.version = '4.0.1'
6
+ s.version = '4.0.2'
7
7
  s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
8
8
  s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
9
9
  s.description = %q{Elasticsearch output plugin for Fluent event collector}
@@ -0,0 +1,70 @@
1
+ require 'openssl'
2
+ require 'fluent/configurable'
3
+ require 'fluent/config/error'
4
+
5
+ module Fluent::Plugin
6
+ module ElasticsearchTLS
7
+ SUPPORTED_TLS_VERSIONS = if defined?(OpenSSL::SSL::TLS1_3_VERSION)
8
+ [:TLSv1, :TLSv1_1, :TLSv1_2, :TLSv1_3].freeze
9
+ else
10
+ [:SSLv23, :TLSv1, :TLSv1_1, :TLSv1_2].freeze
11
+ end
12
+
13
+ DEFAULT_VERSION = :TLSv1
14
+ METHODS_MAP = begin
15
+ # When openssl supports OpenSSL::SSL::TLSXXX constants representations, we use them.
16
+ map = {
17
+ TLSv1: OpenSSL::SSL::TLS1_VERSION,
18
+ TLSv1_1: OpenSSL::SSL::TLS1_1_VERSION,
19
+ TLSv1_2: OpenSSL::SSL::TLS1_2_VERSION
20
+ }
21
+ map[:TLSv1_3] = OpenSSL::SSL::TLS1_3_VERSION if defined?(OpenSSL::SSL::TLS1_3_VERSION)
22
+ USE_TLS_MINMAX_VERSION = true
23
+ map.freeze
24
+ rescue NameError
25
+ map = {
26
+ SSLv23: :SSLv23,
27
+ TLSv1: :TLSv1,
28
+ TLSv1_1: :TLSv1_1,
29
+ TLSv1_2: :TLSv1_2,
30
+ }
31
+ USE_TLS_MINMAX_VERSION = false
32
+ end
33
+ private_constant :METHODS_MAP
34
+
35
+ module ElasticsearchTLSParams
36
+ include Fluent::Configurable
37
+
38
+ config_param :ssl_version, :enum, list: Fluent::Plugin::ElasticsearchTLS::SUPPORTED_TLS_VERSIONS, default: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION
39
+ config_param :ssl_min_version, :enum, list: Fluent::Plugin::ElasticsearchTLS::SUPPORTED_TLS_VERSIONS, default: nil
40
+ config_param :ssl_max_version, :enum, list: Fluent::Plugin::ElasticsearchTLS::SUPPORTED_TLS_VERSIONS, default: nil
41
+ end
42
+
43
+ def self.included(mod)
44
+ mod.include ElasticsearchTLSParams
45
+ end
46
+
47
+ def set_tls_minmax_version_config(ssl_version, ssl_max_version, ssl_min_version)
48
+ if USE_TLS_MINMAX_VERSION
49
+ case
50
+ when ssl_min_version.nil? && ssl_max_version.nil?
51
+ ssl_min_version = METHODS_MAP[ssl_version]
52
+ ssl_max_version = METHODS_MAP[ssl_version]
53
+ when ssl_min_version && ssl_max_version.nil?
54
+ raise Fluent::ConfigError, "When you set 'ssl_min_version', must set 'ssl_max_version' together."
55
+ when ssl_min_version.nil? && ssl_max_version
56
+ raise Fluent::ConfigError, "When you set 'ssl_max_version', must set 'ssl_min_version' together."
57
+ else
58
+ ssl_min_version = METHODS_MAP[ssl_min_version]
59
+ ssl_max_version = METHODS_MAP[ssl_max_version]
60
+ end
61
+
62
+ {max_version: ssl_max_version, min_version: ssl_min_version}
63
+ else
64
+ log.warn "'ssl_min_version' does not have any effect in this environment. Use 'ssl_version' instead." unless ssl_min_version.nil?
65
+ log.warn "'ssl_max_version' does not have any effect in this environment. Use 'ssl_version' instead." unless ssl_max_version.nil?
66
+ {version: ssl_version}
67
+ end
68
+ end
69
+ end
70
+ end
@@ -24,6 +24,7 @@ require_relative 'elasticsearch_error'
24
24
  require_relative 'elasticsearch_error_handler'
25
25
  require_relative 'elasticsearch_index_template'
26
26
  require_relative 'elasticsearch_index_lifecycle_management'
27
+ require_relative 'elasticsearch_tls'
27
28
  begin
28
29
  require_relative 'oj_serializer'
29
30
  rescue LoadError
@@ -53,6 +54,7 @@ module Fluent::Plugin
53
54
 
54
55
  attr_reader :alias_indexes
55
56
  attr_reader :template_names
57
+ attr_reader :ssl_version_options
56
58
 
57
59
  helpers :event_emitter, :compat_parameters, :record_accessor
58
60
 
@@ -104,7 +106,6 @@ EOC
104
106
  config_param :client_cert, :string, :default => nil
105
107
  config_param :client_key_pass, :string, :default => nil, :secret => true
106
108
  config_param :ca_file, :string, :default => nil
107
- config_param :ssl_version, :enum, list: [:SSLv23, :TLSv1, :TLSv1_1, :TLSv1_2], :default => :TLSv1
108
109
  config_param :remove_keys, :string, :default => nil
109
110
  config_param :remove_keys_on_update, :string, :default => ""
110
111
  config_param :remove_keys_on_update_key, :string, :default => nil
@@ -167,6 +168,7 @@ EOC
167
168
  include Fluent::ElasticsearchIndexTemplate
168
169
  include Fluent::Plugin::ElasticsearchConstants
169
170
  include Fluent::Plugin::ElasticsearchIndexLifecycleManagement
171
+ include Fluent::Plugin::ElasticsearchTLS
170
172
 
171
173
  def initialize
172
174
  super
@@ -184,6 +186,7 @@ EOC
184
186
  end
185
187
  @time_parser = create_time_parser
186
188
  @backend_options = backend_options
189
+ @ssl_version_options = set_tls_minmax_version_config(@ssl_version, @ssl_max_version, @ssl_min_version)
187
190
 
188
191
  if @remove_keys
189
192
  @remove_keys = @remove_keys.split(/\s*,\s*/)
@@ -501,6 +504,7 @@ EOC
501
504
  {}
502
505
  end
503
506
  headers = { 'Content-Type' => @content_type.to_s }.merge(@custom_headers).merge(gzip_headers)
507
+ ssl_options = { verify: @ssl_verify, ca_file: @ca_file}.merge(@ssl_version_options)
504
508
 
505
509
  transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
506
510
  options: {
@@ -511,7 +515,7 @@ EOC
511
515
  transport_options: {
512
516
  headers: headers,
513
517
  request: { timeout: @request_timeout },
514
- ssl: { verify: @ssl_verify, ca_file: @ca_file, version: @ssl_version }
518
+ ssl: ssl_options,
515
519
  },
516
520
  http: {
517
521
  user: @user,
@@ -50,6 +50,7 @@ module Fluent::Plugin
50
50
  {}
51
51
  end
52
52
  headers = { 'Content-Type' => @content_type.to_s, }.merge(gzip_headers)
53
+ ssl_options = { verify: @ssl_verify, ca_file: @ca_file}.merge(@ssl_version_options)
53
54
  transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
54
55
  options: {
55
56
  reload_connections: @reload_connections,
@@ -59,7 +60,7 @@ module Fluent::Plugin
59
60
  transport_options: {
60
61
  headers: headers,
61
62
  request: { timeout: @request_timeout },
62
- ssl: { verify: @ssl_verify, ca_file: @ca_file, version: @ssl_version }
63
+ ssl: ssl_options,
63
64
  },
64
65
  http: {
65
66
  user: @user,
@@ -0,0 +1,139 @@
1
+ require_relative '../helper'
2
+ require 'fluent/test/driver/output'
3
+ require 'fluent/plugin/output'
4
+ require 'fluent/plugin/elasticsearch_tls'
5
+
6
+ class TestElasticsearchTLS < Test::Unit::TestCase
7
+
8
+ class TestTLSModuleOutput < Fluent::Plugin::Output
9
+ include Fluent::Plugin::ElasticsearchTLS
10
+
11
+ def initialize
12
+ super
13
+ @emit_streams = []
14
+ end
15
+
16
+ def write(chunk)
17
+ es = Fluent::ArrayEventStream.new
18
+ chunk.each do |time, record|
19
+ es.add(time, record)
20
+ end
21
+ @emit_streams << [tag, es]
22
+ end
23
+ end
24
+
25
+ setup do
26
+ Fluent::Test.setup
27
+ @use_tls_minmax_version = begin
28
+ map = {
29
+ TLSv1: OpenSSL::SSL::TLS1_VERSION,
30
+ TLSv1_1: OpenSSL::SSL::TLS1_1_VERSION,
31
+ TLSv1_2: OpenSSL::SSL::TLS1_2_VERSION
32
+ }
33
+ map[:TLSv1_3] = OpenSSL::SSL::TLS1_3_VERSION if defined?(OpenSSL::SSL::TLS1_3_VERSION)
34
+ true
35
+ rescue NameError
36
+ false
37
+ end
38
+ @enabled_tlsv1_3 = begin
39
+ map = {TLSv1_3: OpenSSL::SSL::TLS1_3_VERSION}
40
+ true
41
+ rescue NameError
42
+ false
43
+ end
44
+ end
45
+
46
+ def driver(conf='')
47
+ Fluent::Test::Driver::Output.new(TestTLSModuleOutput).configure(conf)
48
+ end
49
+
50
+ test 'configure' do
51
+ assert_equal Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION, driver.instance.ssl_version
52
+ assert_nil driver.instance.ssl_max_version
53
+ assert_nil driver.instance.ssl_min_version
54
+ end
55
+
56
+ test 'check USE_TLS_MINMAX_VERSION value' do
57
+ assert_equal @use_tls_minmax_version, Fluent::Plugin::ElasticsearchTLS::USE_TLS_MINMAX_VERSION
58
+ end
59
+
60
+ sub_test_case 'set_tls_minmax_version_config' do
61
+ test 'default' do
62
+ d = driver('')
63
+ ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version, nil, nil)
64
+ if @use_tls_minmax_version
65
+ assert_equal({max_version: OpenSSL::SSL::TLS1_VERSION,
66
+ min_version: OpenSSL::SSL::TLS1_VERSION}, ssl_version_options)
67
+ else
68
+ assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION}, ssl_version_options)
69
+ end
70
+ end
71
+
72
+ test 'errorous cases' do
73
+ if @use_tls_minmax_version
74
+ assert_raise(Fluent::ConfigError) do
75
+ d = driver(%{ssl_max_version TLSv1_2})
76
+ d.instance.set_tls_minmax_version_config(d.instance.ssl_version,
77
+ d.instance.ssl_max_version,
78
+ d.instance.ssl_min_version)
79
+ end
80
+ assert_raise(Fluent::ConfigError) do
81
+ d = driver(%{ssl_min_version TLSv1_2})
82
+ d.instance.set_tls_minmax_version_config(d.instance.ssl_version,
83
+ d.instance.ssl_max_version,
84
+ d.instance.ssl_min_version)
85
+ end
86
+ else
87
+ d1 = driver(%{
88
+ ssl_max_version TLSv1_2
89
+ @log_level info
90
+ })
91
+ d1.instance.set_tls_minmax_version_config(d1.instance.ssl_version,
92
+ d1.instance.ssl_max_version,
93
+ d1.instance.ssl_min_version)
94
+
95
+ d1.logs.any? {|a| a.include?("'ssl_max_version' does not have any effect in this environment.") }
96
+ d2 = driver(%{
97
+ ssl_min_version TLSv1_2
98
+ @log_level info
99
+ })
100
+ d2.instance.set_tls_minmax_version_config(d2.instance.ssl_version,
101
+ d2.instance.ssl_max_version,
102
+ d2.instance.ssl_min_version)
103
+ d2.logs.any? {|a| a.include?("'ssl_min_version' does not have any effect in this environment.") }
104
+ end
105
+ end
106
+
107
+ test 'min_version & max_version' do
108
+ config = %{
109
+ ssl_max_version TLSv1_2
110
+ ssl_min_version TLSv1_1
111
+ }
112
+ d = driver(config)
113
+ ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version,
114
+ d.instance.ssl_max_version,
115
+ d.instance.ssl_min_version)
116
+ if @use_tls_minmax_version
117
+ assert_equal({max_version: OpenSSL::SSL::TLS1_2_VERSION,
118
+ min_version: OpenSSL::SSL::TLS1_1_VERSION}, ssl_version_options)
119
+ else
120
+ assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION}, ssl_version_options)
121
+ end
122
+ end
123
+
124
+ test 'TLSv1.3' do
125
+ omit "openssl gem does not support TLSv1.3" unless @enabled_tlsv1_3
126
+ config = %{
127
+ ssl_max_version TLSv1_3
128
+ ssl_min_version TLSv1_2
129
+ }
130
+ d = driver(config)
131
+ ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version,
132
+ d.instance.ssl_max_version,
133
+ d.instance.ssl_min_version)
134
+ assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
135
+ min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
136
+
137
+ end
138
+ end
139
+ end
@@ -218,7 +218,16 @@ class ElasticsearchOutput < Test::Unit::TestCase
218
218
  assert_equal '/es/', instance.path
219
219
  assert_equal 'john', instance.user
220
220
  assert_equal 'doe', instance.password
221
- assert_equal :TLSv1, instance.ssl_version
221
+ assert_equal Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION, instance.ssl_version
222
+ assert_nil instance.ssl_max_version
223
+ assert_nil instance.ssl_min_version
224
+ if Fluent::Plugin::ElasticsearchTLS::USE_TLS_MINMAX_VERSION
225
+ assert_equal({max_version: OpenSSL::SSL::TLS1_VERSION, min_version: OpenSSL::SSL::TLS1_VERSION},
226
+ instance.ssl_version_options)
227
+ else
228
+ assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION},
229
+ instance.ssl_version_options)
230
+ end
222
231
  assert_nil instance.client_key
223
232
  assert_nil instance.client_cert
224
233
  assert_nil instance.client_key_pass
@@ -97,7 +97,16 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
97
97
  assert_equal 'john', instance.user
98
98
  assert_equal 'doe', instance.password
99
99
  assert_equal '/es/', instance.path
100
- assert_equal :TLSv1, instance.ssl_version
100
+ assert_equal Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION, instance.ssl_version
101
+ assert_nil instance.ssl_max_version
102
+ assert_nil instance.ssl_min_version
103
+ if Fluent::Plugin::ElasticsearchTLS::USE_TLS_MINMAX_VERSION
104
+ assert_equal({max_version: OpenSSL::SSL::TLS1_VERSION, min_version: OpenSSL::SSL::TLS1_VERSION},
105
+ instance.ssl_version_options)
106
+ else
107
+ assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION},
108
+ instance.ssl_version_options)
109
+ end
101
110
  assert_nil instance.client_key
102
111
  assert_nil instance.client_cert
103
112
  assert_nil instance.client_key_pass
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.0.1
4
+ version: 4.0.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - diogo
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2020-01-31 00:00:00.000000000 Z
13
+ date: 2020-02-03 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: fluentd
@@ -157,6 +157,7 @@ files:
157
157
  - lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb
158
158
  - lib/fluent/plugin/elasticsearch_index_template.rb
159
159
  - lib/fluent/plugin/elasticsearch_simple_sniffer.rb
160
+ - lib/fluent/plugin/elasticsearch_tls.rb
160
161
  - lib/fluent/plugin/filter_elasticsearch_genid.rb
161
162
  - lib/fluent/plugin/in_elasticsearch.rb
162
163
  - lib/fluent/plugin/oj_serializer.rb
@@ -166,6 +167,7 @@ files:
166
167
  - test/plugin/test_alias_template.json
167
168
  - test/plugin/test_elasticsearch_error_handler.rb
168
169
  - test/plugin/test_elasticsearch_index_lifecycle_management.rb
170
+ - test/plugin/test_elasticsearch_tls.rb
169
171
  - test/plugin/test_filter_elasticsearch_genid.rb
170
172
  - test/plugin/test_in_elasticsearch.rb
171
173
  - test/plugin/test_out_elasticsearch.rb
@@ -201,6 +203,7 @@ test_files:
201
203
  - test/plugin/test_alias_template.json
202
204
  - test/plugin/test_elasticsearch_error_handler.rb
203
205
  - test/plugin/test_elasticsearch_index_lifecycle_management.rb
206
+ - test/plugin/test_elasticsearch_tls.rb
204
207
  - test/plugin/test_filter_elasticsearch_genid.rb
205
208
  - test/plugin/test_in_elasticsearch.rb
206
209
  - test/plugin/test_out_elasticsearch.rb