fluent-plugin-elasticsearch 5.2.3 → 5.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/issue-auto-closer.yml +1 -1
- data/.github/workflows/linux.yml +1 -1
- data/.github/workflows/macos.yml +1 -1
- data/.github/workflows/windows.yml +1 -1
- data/Gemfile +0 -1
- data/History.md +13 -0
- data/README.md +4 -5
- data/fluent-plugin-elasticsearch.gemspec +2 -1
- data/lib/fluent/plugin/elasticsearch_index_template.rb +2 -2
- data/lib/fluent/plugin/out_elasticsearch_data_stream.rb +14 -3
- data/test/helper.rb +0 -3
- data/test/plugin/test_out_elasticsearch.rb +2 -192
- data/test/plugin/test_out_elasticsearch_data_stream.rb +205 -24
- metadata +17 -4
- data/.coveralls.yml +0 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9e6354575b148be2666c68a6a3c72fcda76139517570203751217cb130fcc070
|
4
|
+
data.tar.gz: 32a0c118516bb5dca54e402e655309565d08f2e1d01caf467a39f2c15d58f26f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 444ef6890245dd454adde27b32478369c913f8a4a4b55bd51025390fcf8850ffceb08203fd85265a3fbbdb9f8f4e7bf77526da2119860de1a6c35fe525275ef9
|
7
|
+
data.tar.gz: a78b3e755f9a9f4eef63fb694cc39ceb6db5bc3d565f46f1934b234d00ddb48ea76235db8623f84e70a64afd844f29dc90b75d4d237194054e0777a92f5d199c
|
@@ -5,7 +5,7 @@ jobs:
|
|
5
5
|
runs-on: ubuntu-latest
|
6
6
|
steps:
|
7
7
|
- name: Autoclose issues that did not follow issue template
|
8
|
-
uses: roots/issue-closer-action@v1.
|
8
|
+
uses: roots/issue-closer-action@v1.2
|
9
9
|
with:
|
10
10
|
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
11
11
|
issue-close-message: "@${issue.user.login} this issue was automatically closed because it did not follow the issue template."
|
data/.github/workflows/linux.yml
CHANGED
data/.github/workflows/macos.yml
CHANGED
data/Gemfile
CHANGED
@@ -4,7 +4,6 @@ source 'https://rubygems.org'
|
|
4
4
|
gemspec
|
5
5
|
|
6
6
|
gem 'simplecov', require: false
|
7
|
-
gem 'coveralls', ">= 0.8.0", require: false
|
8
7
|
gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
|
9
8
|
gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
|
10
9
|
gem "elasticsearch-xpack" if ENV["USE_XPACK"]
|
data/History.md
CHANGED
@@ -2,6 +2,19 @@
|
|
2
2
|
|
3
3
|
### [Unreleased]
|
4
4
|
|
5
|
+
### 5.2.5
|
6
|
+
|
7
|
+
- Update the name of parameter about `thread_pool`.
|
8
|
+
It should be `thread_pool.write.queue_size` instead of `thread_pool.bulk.queue_size` (#1000)
|
9
|
+
- Replace obsolete `File.exists?` with `File.exist?` (#998)
|
10
|
+
- Support `compression_level` for `elasticsearch_data_stream` (#995)
|
11
|
+
- Honor `hosts` parameter for `elasticsearch_data_stream` (#1008)
|
12
|
+
- Drop outdated coveralls (#1009)
|
13
|
+
|
14
|
+
### 5.2.4
|
15
|
+
- Pin Faraday 1.10 (#987)
|
16
|
+
- Increase errors metric on error response in data stream (#986)
|
17
|
+
|
5
18
|
### 5.2.3
|
6
19
|
- Bump actions/checkout from 2 to 3 (#978)
|
7
20
|
- chore: Included githubactions in the dependabot config (#977)
|
data/README.md
CHANGED
@@ -4,7 +4,6 @@
|
|
4
4
|

|
5
5
|

|
6
6
|

|
7
|
-
[](https://coveralls.io/r/uken/fluent-plugin-elasticsearch)
|
8
7
|
[](https://codeclimate.com/github/uken/fluent-plugin-elasticsearch)
|
9
8
|
|
10
9
|
Send your logs to Elasticsearch (and search them with Kibana maybe?)
|
@@ -201,7 +200,7 @@ You can specify Elasticsearch port by this parameter.
|
|
201
200
|
### cloud_id
|
202
201
|
|
203
202
|
```
|
204
|
-
cloud_id test-dep:ZXVyb3BlLXdlc3QxLmdjcC5jbG91ZC5lcy5pbyRiYZTA1Ng==
|
203
|
+
cloud_id test-dep:ZXVyb3BlLXdlc3QxLmdjcC5jbG91ZC5lcy5pbyRiYZTA1Ng==
|
205
204
|
```
|
206
205
|
|
207
206
|
You can specify Elasticsearch cloud_id by this parameter.
|
@@ -1179,11 +1178,11 @@ Advanced users can increase its capacity, but normal users should follow default
|
|
1179
1178
|
|
1180
1179
|
If you want to increase it and forcibly retrying bulk request, please consider to change `unrecoverable_error_types` parameter from default value.
|
1181
1180
|
|
1182
|
-
Change default value of `thread_pool.
|
1181
|
+
Change default value of `thread_pool.write.queue_size` in elasticsearch.yml:
|
1183
1182
|
e.g.)
|
1184
1183
|
|
1185
1184
|
```yaml
|
1186
|
-
thread_pool.
|
1185
|
+
thread_pool.write.queue_size: 1000
|
1187
1186
|
```
|
1188
1187
|
|
1189
1188
|
Then, remove `es_rejected_execution_exception` from `unrecoverable_error_types` parameter:
|
@@ -1531,7 +1530,7 @@ This parameter is mandatory for `elasticsearch_data_stream`.
|
|
1531
1530
|
|
1532
1531
|
### data_stream_template_name
|
1533
1532
|
|
1534
|
-
You can specify an existing matching index template for the data stream. If not present, it creates a new matching index template.
|
1533
|
+
You can specify an existing matching index template for the data stream. If not present, it creates a new matching index template.
|
1535
1534
|
|
1536
1535
|
Default value is `data_stream_name`.
|
1537
1536
|
|
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-elasticsearch'
|
6
|
-
s.version = '5.2.
|
6
|
+
s.version = '5.2.5'
|
7
7
|
s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
|
8
8
|
s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
|
9
9
|
s.description = %q{Elasticsearch output plugin for Fluent event collector}
|
@@ -23,6 +23,7 @@ Gem::Specification.new do |s|
|
|
23
23
|
s.required_ruby_version = Gem::Requirement.new(">= 2.3".freeze)
|
24
24
|
|
25
25
|
s.add_runtime_dependency 'fluentd', '>= 0.14.22'
|
26
|
+
s.add_runtime_dependency "faraday", "~> 1.10"
|
26
27
|
s.add_runtime_dependency 'excon', '>= 0'
|
27
28
|
s.add_runtime_dependency 'elasticsearch'
|
28
29
|
|
@@ -3,7 +3,7 @@ require_relative './elasticsearch_error'
|
|
3
3
|
|
4
4
|
module Fluent::ElasticsearchIndexTemplate
|
5
5
|
def get_template(template_file)
|
6
|
-
if !File.
|
6
|
+
if !File.exist?(template_file)
|
7
7
|
raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
|
8
8
|
end
|
9
9
|
file_contents = IO.read(template_file).gsub(/\n/,'')
|
@@ -11,7 +11,7 @@ module Fluent::ElasticsearchIndexTemplate
|
|
11
11
|
end
|
12
12
|
|
13
13
|
def get_custom_template(template_file, customize_template)
|
14
|
-
if !File.
|
14
|
+
if !File.exist?(template_file)
|
15
15
|
raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
|
16
16
|
end
|
17
17
|
file_contents = IO.read(template_file).gsub(/\n/,'')
|
@@ -219,7 +219,11 @@ module Fluent::Plugin
|
|
219
219
|
data_stream_ilm_name = @data_stream_ilm_name
|
220
220
|
host = nil
|
221
221
|
if @use_placeholder
|
222
|
-
host =
|
222
|
+
host = if @hosts
|
223
|
+
extract_placeholders(@hosts, chunk)
|
224
|
+
else
|
225
|
+
extract_placeholders(@host, chunk)
|
226
|
+
end
|
223
227
|
data_stream_name = extract_placeholders(@data_stream_name, chunk)
|
224
228
|
data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk)
|
225
229
|
data_stream_ilm_name = extract_placeholders(@data_stream_ilm_name, chunk)
|
@@ -257,14 +261,21 @@ module Fluent::Plugin
|
|
257
261
|
end
|
258
262
|
end
|
259
263
|
|
264
|
+
prepared_data = if compression
|
265
|
+
gzip(bulk_message)
|
266
|
+
else
|
267
|
+
bulk_message
|
268
|
+
end
|
269
|
+
|
260
270
|
params = {
|
261
271
|
index: data_stream_name,
|
262
|
-
body:
|
272
|
+
body: prepared_data
|
263
273
|
}
|
264
274
|
begin
|
265
|
-
response = client(host).bulk(params)
|
275
|
+
response = client(host, compression).bulk(params)
|
266
276
|
if response['errors']
|
267
277
|
log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{response}"
|
278
|
+
@num_errors_metrics.inc
|
268
279
|
end
|
269
280
|
rescue => e
|
270
281
|
raise RecoverableRequestFailure, "could not push logs to Elasticsearch cluster (#{data_stream_name}): #{e.message}"
|
data/test/helper.rb
CHANGED
@@ -871,55 +871,6 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
871
871
|
end
|
872
872
|
end
|
873
873
|
|
874
|
-
class GetElasticsearchVersionTest < self
|
875
|
-
def create_driver(conf='', client_version="\"5.0\"")
|
876
|
-
# For request stub to detect compatibility.
|
877
|
-
@client_version ||= client_version
|
878
|
-
# Ensure original implementation existence.
|
879
|
-
Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
|
880
|
-
def detect_es_major_version
|
881
|
-
@_es_info ||= client.info
|
882
|
-
unless version = @_es_info.dig("version", "number")
|
883
|
-
version = @default_elasticsearch_version
|
884
|
-
end
|
885
|
-
version.to_i
|
886
|
-
end
|
887
|
-
CODE
|
888
|
-
Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
|
889
|
-
def client_library_version
|
890
|
-
#{@client_version}
|
891
|
-
end
|
892
|
-
CODE
|
893
|
-
Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput).configure(conf)
|
894
|
-
end
|
895
|
-
|
896
|
-
def test_retry_get_es_version
|
897
|
-
config = %{
|
898
|
-
host logs.google.com
|
899
|
-
port 778
|
900
|
-
scheme https
|
901
|
-
path /es/
|
902
|
-
user john
|
903
|
-
password doe
|
904
|
-
verify_es_version_at_startup true
|
905
|
-
max_retry_get_es_version 3
|
906
|
-
}
|
907
|
-
|
908
|
-
connection_resets = 0
|
909
|
-
stub_request(:get, "https://logs.google.com:778/es//").
|
910
|
-
with(basic_auth: ['john', 'doe']) do |req|
|
911
|
-
connection_resets += 1
|
912
|
-
raise Faraday::ConnectionFailed, "Test message"
|
913
|
-
end
|
914
|
-
|
915
|
-
assert_raise(Fluent::Plugin::ElasticsearchError::RetryableOperationExhaustedFailure) do
|
916
|
-
create_driver(config)
|
917
|
-
end
|
918
|
-
|
919
|
-
assert_equal(4, connection_resets)
|
920
|
-
end
|
921
|
-
end
|
922
|
-
|
923
874
|
class GetElasticsearchIncompatibleVersionTest < self
|
924
875
|
def create_driver(conf='', client_version="7.14")
|
925
876
|
# For request stub to detect compatibility.
|
@@ -981,63 +932,6 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
981
932
|
end
|
982
933
|
end
|
983
934
|
|
984
|
-
class GetElasticsearchVersionWithFallbackTest < self
|
985
|
-
def create_driver(conf='', client_version="\"5.0\"")
|
986
|
-
# For request stub to detect compatibility.
|
987
|
-
@client_version ||= client_version
|
988
|
-
# Ensure original implementation existence.
|
989
|
-
Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
|
990
|
-
def detect_es_major_version
|
991
|
-
@_es_info ||= client.info
|
992
|
-
unless version = @_es_info.dig("version", "number")
|
993
|
-
version = @default_elasticsearch_version
|
994
|
-
end
|
995
|
-
version.to_i
|
996
|
-
end
|
997
|
-
CODE
|
998
|
-
Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
|
999
|
-
def client_library_version
|
1000
|
-
#{@client_version}
|
1001
|
-
end
|
1002
|
-
CODE
|
1003
|
-
Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput).configure(conf)
|
1004
|
-
end
|
1005
|
-
|
1006
|
-
data("Elasticsearch 5" => ["5.0", 5],
|
1007
|
-
"Elasticsearch 6" => ["6.0", 6],
|
1008
|
-
"Elasticsearch 7" => ["7.0", 7],
|
1009
|
-
"Elasticsearch 8" => ["8.0", 8])
|
1010
|
-
def test_retry_get_es_version_without_fail_on_detecting_es_version_retry_exceeded(data)
|
1011
|
-
client_version, es_major_version = data
|
1012
|
-
config = %{
|
1013
|
-
host logs.google.com
|
1014
|
-
port 778
|
1015
|
-
scheme https
|
1016
|
-
path /es/
|
1017
|
-
user john
|
1018
|
-
password doe
|
1019
|
-
verify_es_version_at_startup true
|
1020
|
-
max_retry_get_es_version 2
|
1021
|
-
fail_on_detecting_es_version_retry_exceed false
|
1022
|
-
default_elasticsearch_version #{es_major_version}
|
1023
|
-
@log_level info
|
1024
|
-
}
|
1025
|
-
|
1026
|
-
connection_resets = 0
|
1027
|
-
stub_request(:get, "https://logs.google.com:778/es//").
|
1028
|
-
with(basic_auth: ['john', 'doe']) do |req|
|
1029
|
-
connection_resets += 1
|
1030
|
-
raise Faraday::ConnectionFailed, "Test message"
|
1031
|
-
end
|
1032
|
-
|
1033
|
-
d = create_driver(config, client_version)
|
1034
|
-
|
1035
|
-
assert_equal es_major_version, d.instance.default_elasticsearch_version
|
1036
|
-
assert_equal 3, connection_resets
|
1037
|
-
assert_equal es_major_version, d.instance.instance_variable_get(:@last_seen_major_version)
|
1038
|
-
end
|
1039
|
-
end
|
1040
|
-
|
1041
935
|
data("legacy_template" => [true, "_template"],
|
1042
936
|
"new_template" => [false, "_index_template"])
|
1043
937
|
def test_template_already_present(data)
|
@@ -2143,7 +2037,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
2143
2037
|
|
2144
2038
|
assert_equal ["logstash-tag1", "logstash-tag2"], driver.instance.alias_indexes
|
2145
2039
|
|
2146
|
-
assert_requested(elastic_request)
|
2040
|
+
assert_requested(elastic_request, at_least_times: 1, at_most_times: 2)
|
2147
2041
|
end
|
2148
2042
|
|
2149
2043
|
|
@@ -3056,7 +2950,7 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
3056
2950
|
assert_equal('mylogs-test-tag2', index_cmds.first['index']['_index'])
|
3057
2951
|
assert_equal ["mylogs-test-tag1", "mylogs-test-tag2"], driver.instance.alias_indexes
|
3058
2952
|
|
3059
|
-
assert_requested(elastic_request)
|
2953
|
+
assert_requested(elastic_request, at_least_times: 1, at_most_times: 2)
|
3060
2954
|
end
|
3061
2955
|
|
3062
2956
|
data("legacy_template" => [true, "_template"],
|
@@ -3375,49 +3269,6 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
3375
3269
|
end
|
3376
3270
|
end
|
3377
3271
|
|
3378
|
-
data("legacy_template" => [true, "_template"],
|
3379
|
-
"new_template" => [false, "_index_template"])
|
3380
|
-
def test_template_retry_install_fails(data)
|
3381
|
-
use_legacy_template_flag, endpoint = data
|
3382
|
-
if !use_legacy_template_flag && Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.8.0")
|
3383
|
-
omit "elastisearch-ruby v7.8.0 or later is needed."
|
3384
|
-
end
|
3385
|
-
cwd = File.dirname(__FILE__)
|
3386
|
-
template_file = if use_legacy_template_flag
|
3387
|
-
File.join(cwd, 'test_template.json')
|
3388
|
-
else
|
3389
|
-
File.join(cwd, 'test_index_template.json')
|
3390
|
-
end
|
3391
|
-
|
3392
|
-
config = %{
|
3393
|
-
host logs.google.com
|
3394
|
-
port 778
|
3395
|
-
scheme https
|
3396
|
-
path /es/
|
3397
|
-
user john
|
3398
|
-
password doe
|
3399
|
-
template_name logstash
|
3400
|
-
template_file #{template_file}
|
3401
|
-
max_retry_putting_template 3
|
3402
|
-
use_legacy_template #{use_legacy_template_flag}
|
3403
|
-
}
|
3404
|
-
|
3405
|
-
connection_resets = 0
|
3406
|
-
# check if template exists
|
3407
|
-
stub_request(:get, "https://logs.google.com:778/es//#{endpoint}/logstash")
|
3408
|
-
.with(basic_auth: ['john', 'doe']) do |req|
|
3409
|
-
connection_resets += 1
|
3410
|
-
raise Faraday::ConnectionFailed, "Test message"
|
3411
|
-
end
|
3412
|
-
stub_elastic_info("https://logs.google.com:778/es//")
|
3413
|
-
|
3414
|
-
assert_raise(Fluent::Plugin::ElasticsearchError::RetryableOperationExhaustedFailure) do
|
3415
|
-
driver(config)
|
3416
|
-
end
|
3417
|
-
|
3418
|
-
assert_equal(4, connection_resets)
|
3419
|
-
end
|
3420
|
-
|
3421
3272
|
transport_errors_handled_separately = [TRANSPORT_CLASS::Transport::Errors::NotFound]
|
3422
3273
|
transport_errors = TRANSPORT_CLASS::Transport::Errors.constants.map { |err| [err, TRANSPORT_CLASS::Transport::Errors.const_get(err)] }
|
3423
3274
|
transport_errors_hash = Hash[transport_errors.select { |err| !transport_errors_handled_separately.include?(err[1]) } ]
|
@@ -3460,47 +3311,6 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
3460
3311
|
assert_equal(1, retries)
|
3461
3312
|
end
|
3462
3313
|
|
3463
|
-
data("legacy_template" => [true, "_template"],
|
3464
|
-
"new_template" => [false, "_index_template"])
|
3465
|
-
def test_template_retry_install_does_not_fail(data)
|
3466
|
-
use_legacy_template_flag, endpoint = data
|
3467
|
-
if !use_legacy_template_flag && Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.8.0")
|
3468
|
-
omit "elastisearch-ruby v7.8.0 or later is needed."
|
3469
|
-
end
|
3470
|
-
cwd = File.dirname(__FILE__)
|
3471
|
-
template_file = if use_legacy_template_flag
|
3472
|
-
File.join(cwd, 'test_template.json')
|
3473
|
-
else
|
3474
|
-
File.join(cwd, 'test_index_template.json')
|
3475
|
-
end
|
3476
|
-
|
3477
|
-
config = %{
|
3478
|
-
host logs.google.com
|
3479
|
-
port 778
|
3480
|
-
scheme https
|
3481
|
-
path /es/
|
3482
|
-
user john
|
3483
|
-
password doe
|
3484
|
-
template_name logstash
|
3485
|
-
template_file #{template_file}
|
3486
|
-
max_retry_putting_template 3
|
3487
|
-
fail_on_putting_template_retry_exceed false
|
3488
|
-
use_legacy_template #{use_legacy_template_flag}
|
3489
|
-
}
|
3490
|
-
|
3491
|
-
connection_resets = 0
|
3492
|
-
# check if template exists
|
3493
|
-
stub_request(:get, "https://logs.google.com:778/es//#{endpoint}/logstash")
|
3494
|
-
.with(basic_auth: ['john', 'doe']) do |req|
|
3495
|
-
connection_resets += 1
|
3496
|
-
raise Faraday::ConnectionFailed, "Test message"
|
3497
|
-
end
|
3498
|
-
stub_elastic_info("https://logs.google.com:778/es//")
|
3499
|
-
|
3500
|
-
driver(config)
|
3501
|
-
|
3502
|
-
assert_equal(4, connection_resets)
|
3503
|
-
end
|
3504
3314
|
|
3505
3315
|
data("legacy_template" => [true, "_template"],
|
3506
3316
|
"new_template" => [false, "_index_template"])
|
@@ -57,6 +57,14 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
57
57
|
}.configure(conf)
|
58
58
|
end
|
59
59
|
|
60
|
+
def elasticsearch_transport_layer_decoupling?
|
61
|
+
Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
|
62
|
+
end
|
63
|
+
|
64
|
+
def elastic_transport_layer?
|
65
|
+
Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
|
66
|
+
end
|
67
|
+
|
60
68
|
def sample_data_stream
|
61
69
|
{
|
62
70
|
'data_streams': [
|
@@ -172,6 +180,18 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
172
180
|
stub_data_stream(datastream_name)
|
173
181
|
end
|
174
182
|
|
183
|
+
def stub_elastic_with_store_index_command_counts(url="http://localhost:9200/_bulk")
|
184
|
+
if @index_command_counts == nil
|
185
|
+
@index_command_counts = {}
|
186
|
+
@index_command_counts.default = 0
|
187
|
+
end
|
188
|
+
|
189
|
+
stub_request(:post, url).with do |req|
|
190
|
+
index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
|
191
|
+
@index_command_counts[url] += index_cmds.size
|
192
|
+
end
|
193
|
+
end
|
194
|
+
|
175
195
|
def data_stream_supported?
|
176
196
|
Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.create("7.9.0")
|
177
197
|
end
|
@@ -525,6 +545,114 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
525
545
|
assert_equal '/default_path', host2[:path]
|
526
546
|
end
|
527
547
|
|
548
|
+
def test_configure_compression
|
549
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
550
|
+
|
551
|
+
config = %{
|
552
|
+
data_stream_name foo
|
553
|
+
data_stream_template_name foo_tpl
|
554
|
+
data_stream_ilm_name foo_ilm_policy
|
555
|
+
compression_level best_compression
|
556
|
+
}
|
557
|
+
|
558
|
+
stub_default
|
559
|
+
|
560
|
+
assert_equal true, driver(config).instance.compression
|
561
|
+
end
|
562
|
+
|
563
|
+
def test_check_compression_strategy
|
564
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
565
|
+
|
566
|
+
config = %{
|
567
|
+
data_stream_name foo
|
568
|
+
data_stream_template_name foo_tpl
|
569
|
+
data_stream_ilm_name foo_ilm_policy
|
570
|
+
compression_level best_speed
|
571
|
+
}
|
572
|
+
|
573
|
+
stub_default
|
574
|
+
stub_bulk_feed
|
575
|
+
|
576
|
+
assert_equal Zlib::BEST_SPEED, driver(config).instance.compression_strategy
|
577
|
+
end
|
578
|
+
|
579
|
+
def test_check_content_encoding_header_with_compression
|
580
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
581
|
+
|
582
|
+
config = %{
|
583
|
+
data_stream_name foo
|
584
|
+
data_stream_template_name foo_tpl
|
585
|
+
data_stream_ilm_name foo_ilm_policy
|
586
|
+
compression_level best_compression
|
587
|
+
}
|
588
|
+
|
589
|
+
stub_default
|
590
|
+
stub_request(:post, "http://localhost:9200/foo/_bulk").
|
591
|
+
to_return(status: 200, body: "", headers: {})
|
592
|
+
|
593
|
+
instance = driver(config).instance
|
594
|
+
|
595
|
+
if elastic_transport_layer?
|
596
|
+
assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
|
597
|
+
elsif elasticsearch_transport_layer_decoupling?
|
598
|
+
assert_equal nil, instance.client.transport.transport.options[:transport_options][:headers]["Content-Encoding"]
|
599
|
+
else
|
600
|
+
assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
|
601
|
+
end
|
602
|
+
|
603
|
+
driver.run(default_tag: 'test') do
|
604
|
+
driver.feed(sample_record)
|
605
|
+
end
|
606
|
+
compressable = instance.compressable_connection
|
607
|
+
|
608
|
+
if elastic_transport_layer?
|
609
|
+
assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
|
610
|
+
elsif elasticsearch_transport_layer_decoupling?
|
611
|
+
assert_equal "gzip", instance.client(nil, compressable).transport.transport.options[:transport_options][:headers]["Content-Encoding"]
|
612
|
+
else
|
613
|
+
assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
|
614
|
+
end
|
615
|
+
end
|
616
|
+
|
617
|
+
def test_check_compression_option_is_passed_to_transport
|
618
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
619
|
+
|
620
|
+
config = %{
|
621
|
+
data_stream_name foo
|
622
|
+
data_stream_template_name foo_tpl
|
623
|
+
data_stream_ilm_name foo_ilm_policy
|
624
|
+
compression_level best_compression
|
625
|
+
}
|
626
|
+
|
627
|
+
stub_default
|
628
|
+
stub_request(:post, "http://localhost:9200/foo/_bulk").
|
629
|
+
to_return(status: 200, body: "", headers: {})
|
630
|
+
|
631
|
+
instance = driver(config).instance
|
632
|
+
|
633
|
+
if elastic_transport_layer?
|
634
|
+
assert_equal false, instance.client.transport.options[:compression]
|
635
|
+
elsif elasticsearch_transport_layer_decoupling?
|
636
|
+
assert_equal false, instance.client.transport.transport.options[:compression]
|
637
|
+
else
|
638
|
+
assert_equal false, instance.client.transport.options[:compression]
|
639
|
+
end
|
640
|
+
|
641
|
+
|
642
|
+
driver.run(default_tag: 'test') do
|
643
|
+
driver.feed(sample_record)
|
644
|
+
end
|
645
|
+
compressable = instance.compressable_connection
|
646
|
+
|
647
|
+
if elastic_transport_layer?
|
648
|
+
assert_equal true, instance.client(nil, compressable).transport.options[:compression]
|
649
|
+
elsif elasticsearch_transport_layer_decoupling?
|
650
|
+
assert_equal true, instance.client(nil, compressable).transport.transport.options[:compression]
|
651
|
+
else
|
652
|
+
assert_equal true, instance.client(nil, compressable).transport.options[:compression]
|
653
|
+
end
|
654
|
+
end
|
655
|
+
|
528
656
|
def test_existent_data_stream
|
529
657
|
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
530
658
|
|
@@ -712,40 +840,93 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
|
712
840
|
assert_equal 1, @bulk_records.length
|
713
841
|
end
|
714
842
|
|
715
|
-
def
|
716
|
-
|
843
|
+
def test_placeholder_writes_to_multi_hosts
|
844
|
+
stub_default("foo_bar", "foo_tpl_bar")
|
845
|
+
hosts = [['192.168.33.50', 9201], ['192.168.33.51', 9201], ['192.168.33.52', 9201]]
|
846
|
+
hosts_string = hosts.map {|x| "#{x[0]}:#{x[1]}"}.compact.join(',')
|
847
|
+
hosts.each do |host_info|
|
848
|
+
host, port = host_info
|
849
|
+
stub_elastic_with_store_index_command_counts("http://#{host}:#{port}/foo_bar/_bulk")
|
850
|
+
stub_elastic_info("http://#{host}:#{port}/")
|
851
|
+
stub_request(:get, "http://#{host}:#{port}/_data_stream/foo_bar").
|
852
|
+
to_return(status: 200, body: "", headers: {})
|
853
|
+
end
|
854
|
+
|
855
|
+
conf = config_element(
|
856
|
+
'ROOT', '', {
|
857
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
858
|
+
'data_stream_name' => 'foo_${key1}',
|
859
|
+
'data_stream_template_name' => 'foo_tpl_${key1}',
|
860
|
+
'hosts' => "#{hosts_string}"
|
861
|
+
}, [config_element('buffer', 'tag,key1', {
|
862
|
+
'timekey' => '1d'
|
863
|
+
}, [])])
|
864
|
+
driver(conf).run(default_tag: 'test') do
|
865
|
+
hashes = {
|
866
|
+
'age' => rand(100),
|
867
|
+
'key1' => 'bar'
|
868
|
+
}
|
869
|
+
1000.times do
|
870
|
+
driver.feed(sample_record.merge(hashes))
|
871
|
+
end
|
872
|
+
end
|
873
|
+
|
874
|
+
# @note: we cannot make multi chunks with options (flush_interval, buffer_chunk_limit)
|
875
|
+
# it's Fluentd test driver's constraint
|
876
|
+
# so @index_command_counts.size is always 1
|
877
|
+
assert(@index_command_counts.size > 0, "not working with hosts options")
|
878
|
+
|
879
|
+
total = 0
|
880
|
+
@index_command_counts.each do |_, count|
|
881
|
+
total += count
|
882
|
+
end
|
883
|
+
assert_equal(2000, total)
|
884
|
+
end
|
717
885
|
|
718
|
-
|
719
|
-
|
886
|
+
# gzip compress data
|
887
|
+
def gzip(string, strategy)
|
888
|
+
wio = StringIO.new("w")
|
889
|
+
w_gz = Zlib::GzipWriter.new(wio, strategy = strategy)
|
890
|
+
w_gz.write(string)
|
891
|
+
w_gz.close
|
892
|
+
wio.string
|
893
|
+
end
|
894
|
+
|
895
|
+
def test_writes_to_data_stream_with_compression
|
896
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
720
897
|
|
721
898
|
config = %{
|
722
|
-
host logs.google.com
|
723
|
-
port 778
|
724
|
-
scheme https
|
725
899
|
data_stream_name foo
|
726
|
-
data_stream_ilm_name foo_ilm_policy
|
727
900
|
data_stream_template_name foo_tpl
|
728
|
-
|
729
|
-
|
730
|
-
template_name logstash
|
731
|
-
template_file #{template_file}
|
732
|
-
max_retry_putting_template 3
|
901
|
+
data_stream_ilm_name foo_ilm_policy
|
902
|
+
compression_level default_compression
|
733
903
|
}
|
734
904
|
|
735
|
-
|
736
|
-
|
737
|
-
|
738
|
-
|
739
|
-
|
740
|
-
|
741
|
-
|
742
|
-
|
905
|
+
bodystr = %({
|
906
|
+
"took" : 500,
|
907
|
+
"errors" : false,
|
908
|
+
"items" : [
|
909
|
+
{
|
910
|
+
"create": {
|
911
|
+
"_index" : "fluentd",
|
912
|
+
"_type" : "fluentd"
|
913
|
+
}
|
914
|
+
}
|
915
|
+
]
|
916
|
+
})
|
743
917
|
|
744
|
-
|
745
|
-
|
918
|
+
compressed_body = gzip(bodystr, Zlib::DEFAULT_COMPRESSION)
|
919
|
+
|
920
|
+
stub_default
|
921
|
+
elastic_request = stub_request(:post, "http://localhost:9200/foo/_bulk").
|
922
|
+
to_return(:status => 200, :headers => {'Content-Type' => 'application/json'}, :body => compressed_body)
|
923
|
+
|
924
|
+
driver(config)
|
925
|
+
driver.run(default_tag: 'test') do
|
926
|
+
driver.feed(sample_record)
|
746
927
|
end
|
747
928
|
|
748
|
-
|
929
|
+
assert_requested(elastic_request)
|
749
930
|
end
|
750
931
|
|
751
932
|
def test_doesnt_update_ilm_policy_if_overwrite_unset
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.2.
|
4
|
+
version: 5.2.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- diogo
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date:
|
13
|
+
date: 2023-03-27 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: fluentd
|
@@ -26,6 +26,20 @@ dependencies:
|
|
26
26
|
- - ">="
|
27
27
|
- !ruby/object:Gem::Version
|
28
28
|
version: 0.14.22
|
29
|
+
- !ruby/object:Gem::Dependency
|
30
|
+
name: faraday
|
31
|
+
requirement: !ruby/object:Gem::Requirement
|
32
|
+
requirements:
|
33
|
+
- - "~>"
|
34
|
+
- !ruby/object:Gem::Version
|
35
|
+
version: '1.10'
|
36
|
+
type: :runtime
|
37
|
+
prerelease: false
|
38
|
+
version_requirements: !ruby/object:Gem::Requirement
|
39
|
+
requirements:
|
40
|
+
- - "~>"
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
version: '1.10'
|
29
43
|
- !ruby/object:Gem::Dependency
|
30
44
|
name: excon
|
31
45
|
requirement: !ruby/object:Gem::Requirement
|
@@ -147,7 +161,6 @@ executables: []
|
|
147
161
|
extensions: []
|
148
162
|
extra_rdoc_files: []
|
149
163
|
files:
|
150
|
-
- ".coveralls.yml"
|
151
164
|
- ".editorconfig"
|
152
165
|
- ".github/ISSUE_TEMPLATE/bug_report.md"
|
153
166
|
- ".github/ISSUE_TEMPLATE/feature_request.md"
|
@@ -224,7 +237,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
224
237
|
- !ruby/object:Gem::Version
|
225
238
|
version: '0'
|
226
239
|
requirements: []
|
227
|
-
rubygems_version: 3.
|
240
|
+
rubygems_version: 3.1.6
|
228
241
|
signing_key:
|
229
242
|
specification_version: 4
|
230
243
|
summary: Elasticsearch output plugin for Fluent event collector
|
data/.coveralls.yml
DELETED