fluent-plugin-elasticsearch 4.0.7 → 4.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/linux.yml +26 -0
- data/.github/workflows/macos.yml +26 -0
- data/.github/workflows/windows.yml +26 -0
- data/History.md +5 -0
- data/README.md +5 -1
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/elasticsearch_tls.rb +3 -3
- data/lib/fluent/plugin/out_elasticsearch.rb +8 -4
- data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +6 -4
- data/test/plugin/test_elasticsearch_tls.rb +2 -2
- data/test/plugin/test_out_elasticsearch.rb +21 -3
- data/test/plugin/test_out_elasticsearch_dynamic.rb +21 -3
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1979708479afe32ade449552ade837bc9cfc8acff343cd248008abc579b794d3
|
4
|
+
data.tar.gz: 6ed1a94f6e8553ca72fb75e319ee954f5a8a676ef4ccc0a6a65ca4d041c101c2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e9957264a1ade9aa7e3b7cac60157c0adc9d9b2cd9e6d3d72ce73ca480e6ee32415bb94ad83c95ee7419e6b02b6695c28c39e2696945906e35e25c43421f70ca
|
7
|
+
data.tar.gz: 0efced0623b3312f1d767c82bb3ac9647e0082389bc6457052a4c74f5063a57ba21ff4effe881a18996c0a3f256ade017ec68b9df7423cb5c63869b3e49c1b07
|
@@ -0,0 +1,26 @@
|
|
1
|
+
name: Testing on Ubuntu
|
2
|
+
on:
|
3
|
+
- push
|
4
|
+
- pull_request
|
5
|
+
jobs:
|
6
|
+
build:
|
7
|
+
runs-on: ${{ matrix.os }}
|
8
|
+
strategy:
|
9
|
+
fail-fast: false
|
10
|
+
matrix:
|
11
|
+
ruby: [ '2.4', '2.5', '2.6' ]
|
12
|
+
os:
|
13
|
+
- ubuntu-latest
|
14
|
+
name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
|
15
|
+
steps:
|
16
|
+
- uses: actions/checkout@v2
|
17
|
+
- uses: ruby/setup-ruby@v1
|
18
|
+
with:
|
19
|
+
ruby-version: ${{ matrix.ruby }}
|
20
|
+
- name: unit testing
|
21
|
+
env:
|
22
|
+
CI: true
|
23
|
+
run: |
|
24
|
+
gem install bundler rake
|
25
|
+
bundle install --jobs 4 --retry 3
|
26
|
+
bundle exec rake test
|
@@ -0,0 +1,26 @@
|
|
1
|
+
name: Testing on macOS
|
2
|
+
on:
|
3
|
+
- push
|
4
|
+
- pull_request
|
5
|
+
jobs:
|
6
|
+
build:
|
7
|
+
runs-on: ${{ matrix.os }}
|
8
|
+
strategy:
|
9
|
+
fail-fast: false
|
10
|
+
matrix:
|
11
|
+
ruby: [ '2.4', '2.5', '2.6' ]
|
12
|
+
os:
|
13
|
+
- macOS-latest
|
14
|
+
name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
|
15
|
+
steps:
|
16
|
+
- uses: actions/checkout@v2
|
17
|
+
- uses: ruby/setup-ruby@v1
|
18
|
+
with:
|
19
|
+
ruby-version: ${{ matrix.ruby }}
|
20
|
+
- name: unit testing
|
21
|
+
env:
|
22
|
+
CI: true
|
23
|
+
run: |
|
24
|
+
gem install bundler rake
|
25
|
+
bundle install --jobs 4 --retry 3
|
26
|
+
bundle exec rake test
|
@@ -0,0 +1,26 @@
|
|
1
|
+
name: Testing on Windows
|
2
|
+
on:
|
3
|
+
- push
|
4
|
+
- pull_request
|
5
|
+
jobs:
|
6
|
+
build:
|
7
|
+
runs-on: ${{ matrix.os }}
|
8
|
+
strategy:
|
9
|
+
fail-fast: false
|
10
|
+
matrix:
|
11
|
+
ruby: [ '2.4', '2.5', '2.6' ]
|
12
|
+
os:
|
13
|
+
- windows-latest
|
14
|
+
name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
|
15
|
+
steps:
|
16
|
+
- uses: actions/checkout@v2
|
17
|
+
- uses: ruby/setup-ruby@v1
|
18
|
+
with:
|
19
|
+
ruby-version: ${{ matrix.ruby }}
|
20
|
+
- name: unit testing
|
21
|
+
env:
|
22
|
+
CI: true
|
23
|
+
run: |
|
24
|
+
gem install bundler rake
|
25
|
+
bundle install --jobs 4 --retry 3
|
26
|
+
bundle exec rake test
|
data/History.md
CHANGED
data/README.md
CHANGED
@@ -881,7 +881,11 @@ ssl_min_version TLSv1_2
|
|
881
881
|
|
882
882
|
Elasticsearch plugin will use TLSv1.2 as minimum ssl version and TLSv1.3 as maximum ssl version on transportation with TLS. Note that when they are used in Elastissearch plugin configuration, *`ssl_version` is not used* to set up TLS version.
|
883
883
|
|
884
|
-
If they are *not* specified in the Elasticsearch plugin configuration,
|
884
|
+
If they are *not* specified in the Elasticsearch plugin configuration, `ssl_max_version` and `ssl_min_version` is set up with:
|
885
|
+
|
886
|
+
In Elasticsearch plugin v4.0.8 or later with Ruby 2.5 or later environment, `ssl_max_version` should be `TLSv1_3` and `ssl_min_version` should be `TLSv1_2`.
|
887
|
+
|
888
|
+
From Elasticsearch plugin v4.0.4 to v4.0.7 with Ruby 2.5 or later environment, the value of `ssl_version` will be *used in `ssl_max_version` and `ssl_min_version`*.
|
885
889
|
|
886
890
|
|
887
891
|
### Proxy Support
|
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-elasticsearch'
|
6
|
-
s.version = '4.0.
|
6
|
+
s.version = '4.0.8'
|
7
7
|
s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
|
8
8
|
s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
|
9
9
|
s.description = %q{Elasticsearch output plugin for Fluent event collector}
|
@@ -10,7 +10,7 @@ module Fluent::Plugin
|
|
10
10
|
[:SSLv23, :TLSv1, :TLSv1_1, :TLSv1_2].freeze
|
11
11
|
end
|
12
12
|
|
13
|
-
DEFAULT_VERSION = :
|
13
|
+
DEFAULT_VERSION = :TLSv1_2
|
14
14
|
METHODS_MAP = begin
|
15
15
|
# When openssl supports OpenSSL::SSL::TLSXXX constants representations, we use them.
|
16
16
|
map = {
|
@@ -48,8 +48,8 @@ module Fluent::Plugin
|
|
48
48
|
if USE_TLS_MINMAX_VERSION
|
49
49
|
case
|
50
50
|
when ssl_min_version.nil? && ssl_max_version.nil?
|
51
|
-
ssl_min_version = METHODS_MAP[
|
52
|
-
ssl_max_version = METHODS_MAP[
|
51
|
+
ssl_min_version = METHODS_MAP[:TLSv1_2]
|
52
|
+
ssl_max_version = METHODS_MAP[:TLSv1_3]
|
53
53
|
when ssl_min_version && ssl_max_version.nil?
|
54
54
|
raise Fluent::ConfigError, "When you set 'ssl_min_version', must set 'ssl_max_version' together."
|
55
55
|
when ssl_min_version.nil? && ssl_max_version
|
@@ -55,6 +55,7 @@ module Fluent::Plugin
|
|
55
55
|
attr_reader :alias_indexes
|
56
56
|
attr_reader :template_names
|
57
57
|
attr_reader :ssl_version_options
|
58
|
+
attr_reader :compressable_connection
|
58
59
|
|
59
60
|
helpers :event_emitter, :compat_parameters, :record_accessor, :timer
|
60
61
|
|
@@ -343,6 +344,7 @@ EOC
|
|
343
344
|
@routing_key_name = configure_routing_key_name
|
344
345
|
@meta_config_map = create_meta_config_map
|
345
346
|
@current_config = nil
|
347
|
+
@compressable_connection = false
|
346
348
|
|
347
349
|
@ignore_exception_classes = @ignore_exceptions.map do |exception|
|
348
350
|
unless Object.const_defined?(exception)
|
@@ -511,13 +513,15 @@ EOC
|
|
511
513
|
return Time.at(event_time).to_datetime
|
512
514
|
end
|
513
515
|
|
514
|
-
def client(host = nil)
|
516
|
+
def client(host = nil, compress_connection = false)
|
515
517
|
# check here to see if we already have a client connection for the given host
|
516
518
|
connection_options = get_connection_options(host)
|
517
519
|
|
518
520
|
@_es = nil unless is_existing_connection(connection_options[:hosts])
|
521
|
+
@_es = nil unless @compressable_connection == compress_connection
|
519
522
|
|
520
523
|
@_es ||= begin
|
524
|
+
@compressable_connection = compress_connection
|
521
525
|
@current_config = connection_options[:hosts].clone
|
522
526
|
adapter_conf = lambda {|f| f.adapter @http_backend, @backend_options }
|
523
527
|
local_reload_connections = @reload_connections
|
@@ -525,7 +529,7 @@ EOC
|
|
525
529
|
local_reload_connections = @reload_after
|
526
530
|
end
|
527
531
|
|
528
|
-
gzip_headers = if
|
532
|
+
gzip_headers = if compress_connection
|
529
533
|
{'Content-Encoding' => 'gzip'}
|
530
534
|
else
|
531
535
|
{}
|
@@ -551,7 +555,7 @@ EOC
|
|
551
555
|
},
|
552
556
|
sniffer_class: @sniffer_class,
|
553
557
|
serializer_class: @serializer_class,
|
554
|
-
compression:
|
558
|
+
compression: compress_connection,
|
555
559
|
}), &adapter_conf)
|
556
560
|
Elasticsearch::Client.new transport: transport
|
557
561
|
end
|
@@ -947,7 +951,7 @@ EOC
|
|
947
951
|
data
|
948
952
|
end
|
949
953
|
|
950
|
-
response = client(info.host).bulk body: prepared_data, index: info.index
|
954
|
+
response = client(info.host, compression).bulk body: prepared_data, index: info.index
|
951
955
|
log.on_trace { log.trace "bulk response: #{response}" }
|
952
956
|
|
953
957
|
if response['errors']
|
@@ -35,16 +35,18 @@ module Fluent::Plugin
|
|
35
35
|
end
|
36
36
|
|
37
37
|
|
38
|
-
def client(host = nil)
|
38
|
+
def client(host = nil, compress_connection = false)
|
39
39
|
# check here to see if we already have a client connection for the given host
|
40
40
|
connection_options = get_connection_options(host)
|
41
41
|
|
42
42
|
@_es = nil unless is_existing_connection(connection_options[:hosts])
|
43
|
+
@_es = nil unless @compressable_connection == compress_connection
|
43
44
|
|
44
45
|
@_es ||= begin
|
46
|
+
@compressable_connection = compress_connection
|
45
47
|
@current_config = connection_options[:hosts].clone
|
46
48
|
adapter_conf = lambda {|f| f.adapter @http_backend, @backend_options }
|
47
|
-
gzip_headers = if
|
49
|
+
gzip_headers = if compress_connection
|
48
50
|
{'Content-Encoding' => 'gzip'}
|
49
51
|
else
|
50
52
|
{}
|
@@ -67,7 +69,7 @@ module Fluent::Plugin
|
|
67
69
|
password: @password,
|
68
70
|
scheme: @scheme
|
69
71
|
},
|
70
|
-
compression:
|
72
|
+
compression: compress_connection,
|
71
73
|
}), &adapter_conf)
|
72
74
|
Elasticsearch::Client.new transport: transport
|
73
75
|
end
|
@@ -228,7 +230,7 @@ module Fluent::Plugin
|
|
228
230
|
else
|
229
231
|
data
|
230
232
|
end
|
231
|
-
response = client(host).bulk body: prepared_data, index: index
|
233
|
+
response = client(host, compression).bulk body: prepared_data, index: index
|
232
234
|
if response['errors']
|
233
235
|
log.error "Could not push log to Elasticsearch: #{response}"
|
234
236
|
end
|
@@ -62,8 +62,8 @@ class TestElasticsearchTLS < Test::Unit::TestCase
|
|
62
62
|
d = driver('')
|
63
63
|
ssl_version_options = d.instance.set_tls_minmax_version_config(d.instance.ssl_version, nil, nil)
|
64
64
|
if @use_tls_minmax_version
|
65
|
-
assert_equal({max_version: OpenSSL::SSL::
|
66
|
-
min_version: OpenSSL::SSL::
|
65
|
+
assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION,
|
66
|
+
min_version: OpenSSL::SSL::TLS1_2_VERSION}, ssl_version_options)
|
67
67
|
else
|
68
68
|
assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION}, ssl_version_options)
|
69
69
|
end
|
@@ -230,7 +230,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
230
230
|
assert_nil instance.ssl_max_version
|
231
231
|
assert_nil instance.ssl_min_version
|
232
232
|
if Fluent::Plugin::ElasticsearchTLS::USE_TLS_MINMAX_VERSION
|
233
|
-
assert_equal({max_version: OpenSSL::SSL::
|
233
|
+
assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION, min_version: OpenSSL::SSL::TLS1_2_VERSION},
|
234
234
|
instance.ssl_version_options)
|
235
235
|
else
|
236
236
|
assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION},
|
@@ -284,7 +284,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
284
284
|
}
|
285
285
|
instance = driver(config).instance
|
286
286
|
|
287
|
-
assert_equal
|
287
|
+
assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
|
288
|
+
|
289
|
+
stub_request(:post, "http://localhost:9200/_bulk").
|
290
|
+
to_return(status: 200, body: "", headers: {})
|
291
|
+
driver.run(default_tag: 'test') do
|
292
|
+
driver.feed(sample_record)
|
293
|
+
end
|
294
|
+
compressable = instance.compressable_connection
|
295
|
+
|
296
|
+
assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
|
288
297
|
end
|
289
298
|
|
290
299
|
test 'check compression option is passed to transport' do
|
@@ -295,7 +304,16 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
295
304
|
}
|
296
305
|
instance = driver(config).instance
|
297
306
|
|
298
|
-
assert_equal
|
307
|
+
assert_equal false, instance.client.transport.options[:compression]
|
308
|
+
|
309
|
+
stub_request(:post, "http://localhost:9200/_bulk").
|
310
|
+
to_return(status: 200, body: "", headers: {})
|
311
|
+
driver.run(default_tag: 'test') do
|
312
|
+
driver.feed(sample_record)
|
313
|
+
end
|
314
|
+
compressable = instance.compressable_connection
|
315
|
+
|
316
|
+
assert_equal true, instance.client(nil, compressable).transport.options[:compression]
|
299
317
|
end
|
300
318
|
|
301
319
|
test 'configure Content-Type' do
|
@@ -101,7 +101,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
101
101
|
assert_nil instance.ssl_max_version
|
102
102
|
assert_nil instance.ssl_min_version
|
103
103
|
if Fluent::Plugin::ElasticsearchTLS::USE_TLS_MINMAX_VERSION
|
104
|
-
assert_equal({max_version: OpenSSL::SSL::
|
104
|
+
assert_equal({max_version: OpenSSL::SSL::TLS1_3_VERSION, min_version: OpenSSL::SSL::TLS1_2_VERSION},
|
105
105
|
instance.ssl_version_options)
|
106
106
|
else
|
107
107
|
assert_equal({version: Fluent::Plugin::ElasticsearchTLS::DEFAULT_VERSION},
|
@@ -147,7 +147,16 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
147
147
|
}
|
148
148
|
instance = driver(config).instance
|
149
149
|
|
150
|
-
assert_equal
|
150
|
+
assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
|
151
|
+
|
152
|
+
stub_request(:post, "http://localhost:9200/_bulk").
|
153
|
+
to_return(status: 200, body: "", headers: {})
|
154
|
+
driver.run(default_tag: 'test') do
|
155
|
+
driver.feed(sample_record)
|
156
|
+
end
|
157
|
+
compressable = instance.compressable_connection
|
158
|
+
|
159
|
+
assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
|
151
160
|
end
|
152
161
|
|
153
162
|
test 'check compression option is passed to transport' do
|
@@ -158,7 +167,16 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
158
167
|
}
|
159
168
|
instance = driver(config).instance
|
160
169
|
|
161
|
-
assert_equal
|
170
|
+
assert_equal false, instance.client.transport.options[:compression]
|
171
|
+
|
172
|
+
stub_request(:post, "http://localhost:9200/_bulk").
|
173
|
+
to_return(status: 200, body: "", headers: {})
|
174
|
+
driver.run(default_tag: 'test') do
|
175
|
+
driver.feed(sample_record)
|
176
|
+
end
|
177
|
+
compressable = instance.compressable_connection
|
178
|
+
|
179
|
+
assert_equal true, instance.client(nil, compressable).transport.options[:compression]
|
162
180
|
end
|
163
181
|
|
164
182
|
test 'configure Content-Type' do
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.0.
|
4
|
+
version: 4.0.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- diogo
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2020-
|
13
|
+
date: 2020-05-07 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: fluentd
|
@@ -135,6 +135,9 @@ extra_rdoc_files: []
|
|
135
135
|
files:
|
136
136
|
- ".coveralls.yml"
|
137
137
|
- ".editorconfig"
|
138
|
+
- ".github/workflows/linux.yml"
|
139
|
+
- ".github/workflows/macos.yml"
|
140
|
+
- ".github/workflows/windows.yml"
|
138
141
|
- ".gitignore"
|
139
142
|
- ".travis.yml"
|
140
143
|
- CONTRIBUTING.md
|